From 7642e6917183a002b511f52df8fe2eccca72ed4b Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 5 Jun 2024 14:06:59 +0200 Subject: [PATCH 001/128] use a different struct to implement the balance streaming for utxo it was confusing why a utxo standard coin was being used instead of the coin we are building, this makes a bit more sense --- mm2src/coins/utxo/utxo_balance_events.rs | 50 ++++++++++++------- .../utxo/utxo_builder/utxo_arc_builder.rs | 7 +-- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 2d97ef5cc9..5e3c201488 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -9,7 +9,7 @@ use mm2_event_stream::{behaviour::{EventBehaviour, EventInitStatus}, ErrorEventName, Event, EventName, EventStreamConfiguration}; use std::collections::{BTreeMap, HashSet}; -use super::utxo_standard::UtxoStandardCoin; +use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; use crate::{utxo::{output_script, rpc_clients::electrum_script_hash, utxo_common::{address_balance, address_to_scripthash}, @@ -28,14 +28,29 @@ macro_rules! try_or_continue { }; } +pub struct UtxoBalanceStreamingEvent { + coin: UtxoStandardCoin, +} + +impl UtxoBalanceStreamingEvent { + pub fn new(utxo_arc: UtxoArc) -> Self { + Self { + // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. + // The UtxoArc might belong to a different coin type though. + coin: UtxoStandardCoin::from(utxo_arc), + } + } +} + #[async_trait] -impl EventBehaviour for UtxoStandardCoin { +impl EventBehaviour for UtxoBalanceStreamingEvent { fn event_name() -> EventName { EventName::CoinBalance } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } async fn handle(self, _interval: f64, tx: oneshot::Sender) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let coin = self.coin; async fn subscribe_to_addresses( utxo: &UtxoCoinFields, @@ -74,7 +89,7 @@ impl EventBehaviour for UtxoStandardCoin { Ok(scripthash_to_address_map) } - let ctx = match MmArc::from_weak(&self.as_ref().ctx) { + let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; @@ -84,7 +99,7 @@ impl EventBehaviour for UtxoStandardCoin { }, }; - let scripthash_notification_handler = match self.as_ref().scripthash_notification_handler.as_ref() { + let scripthash_notification_handler = match coin.as_ref().scripthash_notification_handler.as_ref() { Some(t) => t, None => { let e = "Scripthash notification receiver can not be empty."; @@ -101,14 +116,14 @@ impl EventBehaviour for UtxoStandardCoin { let notified_scripthash = match message { ScripthashNotification::Triggered(t) => t, ScripthashNotification::SubscribeToAddresses(addresses) => { - match subscribe_to_addresses(self.as_ref(), addresses).await { + match subscribe_to_addresses(coin.as_ref(), addresses).await { Ok(map) => scripthash_to_address_map.extend(map), Err(e) => { log::error!("{e}"); ctx.stream_channel_controller .broadcast(Event::new( - format!("{}:{}", Self::error_event_name(), self.ticker()), + format!("{}:{}", Self::error_event_name(), coin.ticker()), json!({ "error": e }).to_string(), )) .await; @@ -118,15 +133,15 @@ impl EventBehaviour for UtxoStandardCoin { continue; }, ScripthashNotification::RefreshSubscriptions => { - let my_addresses = try_or_continue!(self.all_addresses().await); - match subscribe_to_addresses(self.as_ref(), my_addresses).await { + let my_addresses = try_or_continue!(coin.all_addresses().await); + match subscribe_to_addresses(coin.as_ref(), my_addresses).await { Ok(map) => scripthash_to_address_map = map, Err(e) => { log::error!("{e}"); ctx.stream_channel_controller .broadcast(Event::new( - format!("{}:{}", Self::error_event_name(), self.ticker()), + format!("{}:{}", Self::error_event_name(), coin.ticker()), json!({ "error": e }).to_string(), )) .await; @@ -139,7 +154,7 @@ impl EventBehaviour for UtxoStandardCoin { let address = match scripthash_to_address_map.get(¬ified_scripthash) { Some(t) => Some(t.clone()), - None => try_or_continue!(self.all_addresses().await) + None => try_or_continue!(coin.all_addresses().await) .into_iter() .find_map(|addr| { let script = match output_script(&addr) { @@ -172,10 +187,10 @@ impl EventBehaviour for UtxoStandardCoin { }, }; - let balance = match address_balance(&self, &address).await { + let balance = match address_balance(&coin, &address).await { Ok(t) => t, Err(e) => { - let ticker = self.ticker(); + let ticker = coin.ticker(); log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); @@ -191,7 +206,7 @@ impl EventBehaviour for UtxoStandardCoin { }; let payload = json!({ - "ticker": self.ticker(), + "ticker": coin.ticker(), "address": address.to_string(), "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); @@ -210,18 +225,19 @@ impl EventBehaviour for UtxoStandardCoin { log::info!( "{} event is activated for {}. `stream_interval_seconds`({}) has no effect on this.", Self::event_name(), - self.ticker(), + self.coin.ticker(), event.stream_interval_seconds ); let (tx, rx): (Sender, Receiver) = oneshot::channel(); - let fut = self.clone().handle(event.stream_interval_seconds, tx); let settings = AbortSettings::info_on_abort(format!( "{} event is stopped for {}.", Self::event_name(), - self.ticker() + self.coin.ticker() )); - self.spawner().spawn_with_settings(fut, settings); + self.coin + .spawner() + .spawn_with_settings(self.handle(event.stream_interval_seconds, tx), settings); rx.await.unwrap_or_else(|e| { EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 60b4d75ff0..73962eb618 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,9 +1,9 @@ use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, UtxoJsonRpcClientInfo, UtxoRpcClientEnum}; +use crate::utxo::utxo_balance_events::UtxoBalanceStreamingEvent; use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithGlobalHDBuilder, UtxoFieldsWithHardwareWalletBuilder, UtxoFieldsWithIguanaSecretBuilder}; -use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::utxo::{generate_and_send_tx, FeePolicy, GetUtxoListOps, UtxoArc, UtxoCommonOps, UtxoSyncStatusLoopHandle, UtxoWeak}; use crate::{DerivationMethod, PrivKeyBuildPolicy, UtxoActivationParams}; @@ -126,8 +126,9 @@ where return MmError::err(UtxoCoinBuildError::UnsupportedModeForBalanceEvents { mode }); } - if let EventInitStatus::Failed(err) = - EventBehaviour::spawn_if_active(UtxoStandardCoin::from(utxo_arc), stream_config).await + if let EventInitStatus::Failed(err) = UtxoBalanceStreamingEvent::new(utxo_arc) + .spawn_if_active(stream_config) + .await { return MmError::err(UtxoCoinBuildError::FailedSpawningBalanceEvents(err)); } From 8aee75b5438deb639d1fd5be17ae6f26e8b96f3e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 17 Jun 2024 13:35:09 +0200 Subject: [PATCH 002/128] rename UtxoBalanceStreaming -> UtxoBalanceEventStreamer --- mm2src/coins/utxo/utxo_balance_events.rs | 8 +++++--- mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 5e3c201488..de72cd3f76 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -28,11 +28,11 @@ macro_rules! try_or_continue { }; } -pub struct UtxoBalanceStreamingEvent { +pub struct UtxoBalanceEventStreamer { coin: UtxoStandardCoin, } -impl UtxoBalanceStreamingEvent { +impl UtxoBalanceEventStreamer { pub fn new(utxo_arc: UtxoArc) -> Self { Self { // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. @@ -43,11 +43,12 @@ impl UtxoBalanceStreamingEvent { } #[async_trait] -impl EventBehaviour for UtxoBalanceStreamingEvent { +impl EventBehaviour for UtxoBalanceEventStreamer { fn event_name() -> EventName { EventName::CoinBalance } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } + // FIXME: Move `interval` to `self`. async fn handle(self, _interval: f64, tx: oneshot::Sender) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; @@ -194,6 +195,7 @@ impl EventBehaviour for UtxoBalanceStreamingEvent { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); + // FIXME: Note that such an event isn't SSE-ed to any client since no body is listening to it. ctx.stream_channel_controller .broadcast(Event::new( format!("{}:{}", Self::error_event_name(), ticker), diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 73962eb618..ad1d929e1a 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,5 +1,5 @@ use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, UtxoJsonRpcClientInfo, UtxoRpcClientEnum}; -use crate::utxo::utxo_balance_events::UtxoBalanceStreamingEvent; +use crate::utxo::utxo_balance_events::UtxoBalanceEventStreamer; use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithGlobalHDBuilder, UtxoFieldsWithHardwareWalletBuilder, @@ -122,11 +122,13 @@ where } if let Some(stream_config) = &self.ctx().event_stream_configuration { + // FIXME: This will fail whenever the coin is in native mode and event streaming is enabled, + // even if balance streaming isn't enabled. if is_native_mode { return MmError::err(UtxoCoinBuildError::UnsupportedModeForBalanceEvents { mode }); } - if let EventInitStatus::Failed(err) = UtxoBalanceStreamingEvent::new(utxo_arc) + if let EventInitStatus::Failed(err) = UtxoBalanceEventStreamer::new(utxo_arc) .spawn_if_active(stream_config) .await { From 94fd9b0b52b547af53ed767e88259bcf27b1c413 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 17 Jun 2024 13:35:43 +0200 Subject: [PATCH 003/128] add fixme: reminder to send sse in running swap --- mm2src/mm2_main/src/lp_swap/maker_swap.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index b1117e707a..2594d614ad 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -2098,11 +2098,9 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { swap_ctx.running_swaps.lock().unwrap().push(weak_ref); let mut swap_fut = Box::pin( async move { - let mut events; loop { let res = running_swap.handle_command(command).await.expect("!handle_command"); - events = res.1; - for event in events { + for event in res.1 { let to_save = MakerSavedEvent { timestamp: now_ms(), event: event.clone(), @@ -2118,6 +2116,7 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { save_my_maker_swap_event(&ctx, &running_swap, to_save) .await .expect("!save_my_maker_swap_event"); + // FIXME: send SSE here if event.should_ban_taker() { ban_pubkey_on_failed_swap( &ctx, From c6ced79c235f461879ddc25e879a39e8491db647 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 27 Jun 2024 19:35:54 +0200 Subject: [PATCH 004/128] refactor(mm2_event_stream::controller): use less types for easier understanding --- mm2src/mm2_event_stream/src/controller.rs | 95 ++++++++++------------- mm2src/mm2_net/src/sse_handler.rs | 2 +- 2 files changed, 41 insertions(+), 56 deletions(-) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 72870308b4..692ba3f60a 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -14,24 +14,15 @@ impl Clone for Controller { /// Inner part of the controller pub struct ChannelsInner { last_id: u64, - channels: HashMap>, + channels: HashMap>>, } -struct Channel { - tx: Sender>, -} - -/// guard to trace channels disconnection -pub struct ChannelGuard { - channel_id: ChannelId, - controller: Controller, -} - -/// Receiver to cleanup resources on `Drop` -pub struct GuardedReceiver { +/// A smart channel receiver that will run a hook when it's dropped to remove itself from the controller. +pub struct ChannelReceiver { + /// The receiver end of the channel. rx: Receiver>, - #[allow(dead_code)] - guard: ChannelGuard, + /// A hook that's ran when the receiver channel receiver is dropped. + channel_drop_hook: Box, } impl Controller { @@ -39,17 +30,23 @@ impl Controller { pub fn new() -> Self { Default::default() } /// Creates a new channel and returns it's events receiver - pub fn create_channel(&mut self, concurrency: usize) -> GuardedReceiver { + pub fn create_channel(&self, concurrency: usize) -> ChannelReceiver { let (tx, rx) = mpsc::channel::>(concurrency); - let channel = Channel { tx }; let mut inner = self.0.lock(); let channel_id = inner.last_id.overflowing_add(1).0; - inner.channels.insert(channel_id, channel); + inner.channels.insert(channel_id, tx); inner.last_id = channel_id; - let guard = ChannelGuard::new(channel_id, self.clone()); - GuardedReceiver { rx, guard } + let channel_drop_hook = { + let controller = self.clone(); + // Remove the channel from the controller when the receiver is dropped. + Box::new(move || { + common::log::debug!("Dropping event receiver"); + controller.remove_channel(&channel_id) + }) + }; + ChannelReceiver { rx, channel_drop_hook } } /// Returns number of active channels @@ -58,19 +55,19 @@ impl Controller { /// Broadcast message to all channels pub async fn broadcast(&self, message: M) { let msg = Arc::new(message); - for rx in self.all_senders() { - rx.send(Arc::clone(&msg)).await.ok(); + for tx in self.all_channels() { + tx.send(Arc::clone(&msg)).await.ok(); } } /// Removes the channel from the controller - fn remove_channel(&mut self, channel_id: &ChannelId) { + fn remove_channel(&self, channel_id: &ChannelId) { let mut inner = self.0.lock(); inner.channels.remove(channel_id); } - /// Returns all the active channels - fn all_senders(&self) -> Vec>> { self.0.lock().channels.values().map(|c| c.tx.clone()).collect() } + /// Returns all the active channel senders + fn all_channels(&self) -> Vec>> { self.0.lock().channels.values().cloned().collect() } } impl Default for Controller { @@ -83,23 +80,15 @@ impl Default for Controller { } } -impl ChannelGuard { - fn new(channel_id: ChannelId, controller: Controller) -> Self { Self { channel_id, controller } } -} - -impl Drop for ChannelGuard { - fn drop(&mut self) { - common::log::debug!("Dropping event channel with id: {}", self.channel_id); - - self.controller.remove_channel(&self.channel_id); - } -} - -impl GuardedReceiver { +impl ChannelReceiver { /// Receives the next event from the channel pub async fn recv(&mut self) -> Option> { self.rx.recv().await } } +impl Drop for ChannelReceiver { + fn drop(&mut self) { (self.channel_drop_hook)(); } +} + #[cfg(any(test, target_arch = "wasm32"))] mod tests { use super::*; @@ -111,17 +100,17 @@ mod tests { } cross_test!(test_create_channel_and_broadcast, { - let mut controller = Controller::new(); - let mut guard_receiver = controller.create_channel(1); + let controller = Controller::new(); + let mut channel_receiver = controller.create_channel(1); controller.broadcast("Message".to_string()).await; - let received_msg = guard_receiver.recv().await.unwrap(); + let received_msg = channel_receiver.recv().await.unwrap(); assert_eq!(*received_msg, "Message".to_string()); }); cross_test!(test_multiple_channels_and_broadcast, { - let mut controller = Controller::new(); + let controller = Controller::new(); let mut receivers = Vec::new(); for _ in 0..3 { @@ -137,20 +126,18 @@ mod tests { }); cross_test!(test_channel_cleanup_on_drop, { - let mut controller: Controller<()> = Controller::new(); - let guard_receiver = controller.create_channel(1); + let controller: Controller<()> = Controller::new(); + let channel_receiver = controller.create_channel(1); assert_eq!(controller.num_connections(), 1); - drop(guard_receiver); - - common::executor::Timer::sleep(0.1).await; // Give time for the drop to execute + drop(channel_receiver); assert_eq!(controller.num_connections(), 0); }); cross_test!(test_broadcast_across_channels, { - let mut controller = Controller::new(); + let controller = Controller::new(); let mut receivers = Vec::new(); for _ in 0..3 { @@ -166,8 +153,8 @@ mod tests { }); cross_test!(test_multiple_messages_and_drop, { - let mut controller = Controller::new(); - let mut guard_receiver = controller.create_channel(6); + let controller = Controller::new(); + let mut channel_receiver = controller.create_channel(6); controller.broadcast("Message 1".to_string()).await; controller.broadcast("Message 2".to_string()).await; @@ -178,7 +165,7 @@ mod tests { let mut received_msgs = Vec::new(); for _ in 0..6 { - let received_msg = guard_receiver.recv().await.unwrap(); + let received_msg = channel_receiver.recv().await.unwrap(); received_msgs.push(received_msg); } @@ -189,10 +176,8 @@ mod tests { assert_eq!(*received_msgs[4], "Message 5".to_string()); assert_eq!(*received_msgs[5], "Message 6".to_string()); - // Consume the GuardedReceiver to trigger drop and channel cleanup - drop(guard_receiver); - - common::executor::Timer::sleep(0.1).await; // Give time for the drop to execute + // Drop the channel receiver to trigger the drop hook (remove itself from the controller). + drop(channel_receiver); assert_eq!(controller.num_connections(), 0); }); diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index 3b3afeee58..f35cd4d32b 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -37,7 +37,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Thu, 27 Jun 2024 19:46:25 +0200 Subject: [PATCH 005/128] add a note about the broadcasting mechanisim --- mm2src/mm2_event_stream/src/controller.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 692ba3f60a..cf655d73c2 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -56,6 +56,9 @@ impl Controller { pub async fn broadcast(&self, message: M) { let msg = Arc::new(message); for tx in self.all_channels() { + // FIXME: We are `awaiting` a back-pressured channel here. If one channel is slow for + // whatever reason, it will block the broadcast to all other channels. + // either `try_send`, `send` with a short timeout, or make the channels unbounded (security risk?). tx.send(Arc::clone(&msg)).await.ok(); } } From 876226b2a0647f0b1d5fc89c6036e65ec0831874 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 27 Jun 2024 23:00:54 +0200 Subject: [PATCH 006/128] Take `event_stream_configuration` out of the `Option<` There was no point of the option wrapping. Simplifying stuff. --- mm2src/coins/eth.rs | 8 ++-- mm2src/coins/utxo/utxo_balance_events.rs | 7 +++ .../utxo/utxo_builder/utxo_arc_builder.rs | 19 +++----- .../utxo/utxo_builder/utxo_coin_builder.rs | 44 +++++-------------- mm2src/coins/z_coin.rs | 12 +++-- .../src/platform_coin_with_tokens.rs | 6 +-- mm2src/mm2_core/src/mm_ctx.rs | 10 ++--- mm2src/mm2_main/src/lp_native_dex.rs | 18 +++++--- mm2src/mm2_main/src/rpc.rs | 4 +- mm2src/mm2_net/src/sse_handler.rs | 10 +---- mm2src/mm2_net/src/wasm_event_stream.rs | 5 +-- 11 files changed, 53 insertions(+), 90 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 8ab8b3f113..83e6e1d54c 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5282,10 +5282,10 @@ impl EthCoin { } async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - if let Some(stream_config) = &ctx.event_stream_configuration { - if let EventInitStatus::Failed(err) = EventBehaviour::spawn_if_active(self.clone(), stream_config).await { - return ERR!("Failed spawning balance events. Error: {}", err); - } + if let EventInitStatus::Failed(err) = + EventBehaviour::spawn_if_active(self.clone(), &ctx.event_stream_configuration).await + { + return ERR!("Failed spawning balance events. Error: {}", err); } Ok(()) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index de72cd3f76..835e0b9f03 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -90,6 +90,13 @@ impl EventBehaviour for UtxoBalanceEventStreamer { Ok(scripthash_to_address_map) } + if coin.as_ref().rpc_client.is_native() { + let e = "Native mode is not supported for event streaming"; + tx.send(EventInitStatus::Failed(e.to_string())) + .expect(RECEIVER_DROPPED_MSG); + panic!("{}", e); + } + let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { Some(ctx) => ctx, None => { diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index ad1d929e1a..504219677f 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -109,7 +109,6 @@ where let utxo = self.build_utxo_fields().await?; let sync_status_loop_handle = utxo.block_headers_status_notifier.clone(); let spv_conf = utxo.conf.spv_conf.clone(); - let (is_native_mode, mode) = (utxo.rpc_client.is_native(), utxo.rpc_client.to_string()); let utxo_arc = UtxoArc::new(utxo); self.spawn_merge_utxo_loop_if_required(&utxo_arc, self.constructor.clone()); @@ -121,19 +120,11 @@ where spawn_block_header_utxo_loop(self.ticker, &utxo_arc, sync_handle, spv_conf); } - if let Some(stream_config) = &self.ctx().event_stream_configuration { - // FIXME: This will fail whenever the coin is in native mode and event streaming is enabled, - // even if balance streaming isn't enabled. - if is_native_mode { - return MmError::err(UtxoCoinBuildError::UnsupportedModeForBalanceEvents { mode }); - } - - if let EventInitStatus::Failed(err) = UtxoBalanceEventStreamer::new(utxo_arc) - .spawn_if_active(stream_config) - .await - { - return MmError::err(UtxoCoinBuildError::FailedSpawningBalanceEvents(err)); - } + if let EventInitStatus::Failed(err) = UtxoBalanceEventStreamer::new(utxo_arc) + .spawn_if_active(&self.ctx().event_stream_configuration) + .await + { + return MmError::err(UtxoCoinBuildError::FailedSpawningBalanceEvents(err)); } Ok(result_coin) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 446cadf2bb..2cbb57b35c 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -6,9 +6,8 @@ use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError}; use crate::utxo::{output_script, ElectrumBuilderArgs, ElectrumProtoVerifier, ElectrumProtoVerifierEvent, - RecentlySpentOutPoints, ScripthashNotification, ScripthashNotificationSender, TxFee, UtxoCoinConf, - UtxoCoinFields, UtxoHDWallet, UtxoRpcMode, UtxoSyncStatus, UtxoSyncStatusLoopHandle, - UTXO_DUST_AMOUNT}; + RecentlySpentOutPoints, ScripthashNotificationSender, TxFee, UtxoCoinConf, UtxoCoinFields, + UtxoHDWallet, UtxoRpcMode, UtxoSyncStatus, UtxoSyncStatusLoopHandle, UTXO_DUST_AMOUNT}; use crate::{BlockchainNetwork, CoinTransportMetrics, DerivationMethod, HistorySyncState, IguanaPrivKey, PrivKeyBuildPolicy, PrivKeyPolicy, PrivKeyPolicyNotAllowed, RpcClientType, UtxoActivationParams}; use async_trait::async_trait; @@ -20,7 +19,7 @@ use common::log::{error, info, LogOnError}; use common::{now_sec, small_rng}; use crypto::{Bip32DerPathError, CryptoCtx, CryptoCtxError, GlobalHDAccountArc, HwWalletType, StandardHDPathError}; use derive_more::Display; -use futures::channel::mpsc::{channel, unbounded, Receiver as AsyncReceiver, UnboundedReceiver, UnboundedSender}; +use futures::channel::mpsc::{channel, unbounded, Receiver as AsyncReceiver, UnboundedReceiver}; use futures::compat::Future01CompatExt; use futures::lock::Mutex as AsyncMutex; use futures::StreamExt; @@ -243,25 +242,6 @@ pub trait UtxoFieldsWithGlobalHDBuilder: UtxoCoinBuilderCommonOps { fn gap_limit(&self) -> u32 { self.activation_params().gap_limit.unwrap_or(DEFAULT_GAP_LIMIT) } } -// The return type is one-time used only. No need to create a type for it. -#[allow(clippy::type_complexity)] -fn get_scripthash_notification_handlers( - ctx: &MmArc, -) -> Option<( - UnboundedSender, - Arc>>, -)> { - if ctx.event_stream_configuration.is_some() { - let (sender, receiver): ( - UnboundedSender, - UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - Some((sender, Arc::new(AsyncMutex::new(receiver)))) - } else { - None - } -} - async fn build_utxo_coin_fields_with_conf_and_policy( builder: &Builder, conf: UtxoCoinConf, @@ -285,11 +265,10 @@ where let my_script_pubkey = output_script(&my_address).map(|script| script.to_bytes())?; - let (scripthash_notification_sender, scripthash_notification_handler) = - match get_scripthash_notification_handlers(builder.ctx()) { - Some((sender, receiver)) => (Some(sender), Some(receiver)), - None => (None, None), - }; + let (scripthash_notification_sender, scripthash_notification_handler) = { + let (sender, receiver) = futures::channel::mpsc::unbounded(); + (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) + }; // Create an abortable system linked to the `MmCtx` so if the context is stopped via `MmArc::stop`, // all spawned futures related to this `UTXO` coin will be aborted as well. @@ -372,11 +351,10 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { address_format, }; - let (scripthash_notification_sender, scripthash_notification_handler) = - match get_scripthash_notification_handlers(self.ctx()) { - Some((sender, receiver)) => (Some(sender), Some(receiver)), - None => (None, None), - }; + let (scripthash_notification_sender, scripthash_notification_handler) = { + let (sender, receiver) = futures::channel::mpsc::unbounded(); + (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) + }; // Create an abortable system linked to the `MmCtx` so if the context is stopped via `MmArc::stop`, // all spawned futures related to this `UTXO` coin will be aborted as well. diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 81d0aca85c..3f1c5c7265 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -662,10 +662,10 @@ impl ZCoin { async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { let coin = self.clone(); - if let Some(stream_config) = &ctx.event_stream_configuration { - if let EventInitStatus::Failed(err) = EventBehaviour::spawn_if_active(coin, stream_config).await { - return ERR!("Failed spawning zcoin balance event with error: {}", err); - } + if let EventInitStatus::Failed(err) = + EventBehaviour::spawn_if_active(coin, &ctx.event_stream_configuration).await + { + return ERR!("Failed spawning zcoin balance event with error: {}", err); } Ok(()) @@ -891,11 +891,9 @@ impl<'a> UtxoCoinBuilder for ZCoinBuilder<'a> { ); let blocks_db = self.init_blocks_db().await?; - let (z_balance_event_sender, z_balance_event_handler) = if self.ctx.event_stream_configuration.is_some() { + let (z_balance_event_sender, z_balance_event_handler) = { let (sender, receiver) = futures::channel::mpsc::unbounded(); (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) - } else { - (None, None) }; let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index 051dd22fc3..0d031e7808 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -481,9 +481,9 @@ where ); } - if let Some(config) = &ctx.event_stream_configuration { - platform_coin.handle_balance_streaming(config).await?; - } + platform_coin + .handle_balance_streaming(&ctx.event_stream_configuration) + .await?; let coins_ctx = CoinsContext::from_ctx(&ctx).unwrap(); coins_ctx diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index af0a6adece..f22d0d12b2 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -81,7 +81,7 @@ pub struct MmCtx { /// Data transfer bridge between server and client where server (which is the mm2 runtime) initiates the request. pub(crate) data_asker: DataAsker, /// Configuration of event streaming used for SSE. - pub event_stream_configuration: Option, + pub event_stream_configuration: EventStreamConfiguration, /// True if the MarketMaker instance needs to stop. pub stop: Constructible, /// Unique context identifier, allowing us to more easily pass the context through the FFI boundaries. @@ -155,7 +155,7 @@ impl MmCtx { rpc_started: Constructible::default(), stream_channel_controller: Controller::new(), data_asker: DataAsker::default(), - event_stream_configuration: None, + event_stream_configuration: Default::default(), stop: Constructible::default(), ffi_handle: Constructible::default(), ordermatch_ctx: Mutex::new(None), @@ -757,10 +757,8 @@ impl MmCtxBuilder { let event_stream_configuration = &ctx.conf["event_stream_configuration"]; if !event_stream_configuration.is_null() { - let event_stream_configuration: EventStreamConfiguration = - json::from_value(event_stream_configuration.clone()) - .expect("Invalid json value in 'event_stream_configuration'."); - ctx.event_stream_configuration = Some(event_stream_configuration); + ctx.event_stream_configuration = json::from_value(event_stream_configuration.clone()) + .expect("Invalid json value in 'event_stream_configuration'."); } } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index dd6ea79caa..bdc8079381 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -429,14 +429,18 @@ fn migration_1(_ctx: &MmArc) {} async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { // This condition only executed if events were enabled in mm2 configuration. - if let Some(config) = &ctx.event_stream_configuration { - if let EventInitStatus::Failed(err) = NetworkEvent::new(ctx.clone()).spawn_if_active(config).await { - return MmError::err(MmInitError::NetworkEventInitFailed(err)); - } + if let EventInitStatus::Failed(err) = NetworkEvent::new(ctx.clone()) + .spawn_if_active(&ctx.event_stream_configuration) + .await + { + return MmError::err(MmInitError::NetworkEventInitFailed(err)); + } - if let EventInitStatus::Failed(err) = HeartbeatEvent::new(ctx.clone()).spawn_if_active(config).await { - return MmError::err(MmInitError::HeartbeatEventInitFailed(err)); - } + if let EventInitStatus::Failed(err) = HeartbeatEvent::new(ctx.clone()) + .spawn_if_active(&ctx.event_stream_configuration) + .await + { + return MmError::err(MmInitError::HeartbeatEventInitFailed(err)); } Ok(()) diff --git a/mm2src/mm2_main/src/rpc.rs b/mm2src/mm2_main/src/rpc.rs index 92422738f9..b99a9a9ec3 100644 --- a/mm2src/mm2_main/src/rpc.rs +++ b/mm2src/mm2_main/src/rpc.rs @@ -359,11 +359,9 @@ pub extern "C" fn spawn_rpc(ctx_h: u32) { let ctx = MmArc::from_ffi_handle(ctx_h).expect("No context"); - let is_event_stream_enabled = ctx.event_stream_configuration.is_some(); - let make_svc_fut = move |remote_addr: SocketAddr| async move { Ok::<_, Infallible>(service_fn(move |req: Request| async move { - if is_event_stream_enabled && req.uri().path() == SSE_ENDPOINT { + if req.uri().path() == SSE_ENDPOINT { let res = handle_sse(req, ctx_h).await?; return Ok::<_, Infallible>(res); } diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index f35cd4d32b..0b05f69870 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -14,15 +14,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result return handle_internal_error(err).await, }; - let config = match &ctx.event_stream_configuration { - Some(config) => config, - None => { - return handle_internal_error( - "Event stream configuration couldn't be found. This should never happen.".to_string(), - ) - .await - }, - }; + let config = &ctx.event_stream_configuration; let filtered_events = request .uri() diff --git a/mm2src/mm2_net/src/wasm_event_stream.rs b/mm2src/mm2_net/src/wasm_event_stream.rs index dcd6da33e2..f3e381cd3d 100644 --- a/mm2src/mm2_net/src/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/wasm_event_stream.rs @@ -12,10 +12,7 @@ unsafe impl Send for SendableMessagePort {} /// Handles broadcasted messages from `mm2_event_stream` continuously for WASM. pub async fn handle_worker_stream(ctx: MmArc) { - let config = ctx - .event_stream_configuration - .as_ref() - .expect("Event stream configuration couldn't be found. This should never happen."); + let config = &ctx.event_stream_configuration; let mut channel_controller = ctx.stream_channel_controller.clone(); let mut rx = channel_controller.create_channel(config.total_active_events()); From 58d436c37102501390a38f239ab741ab6630c6f1 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 28 Jun 2024 03:16:37 +0200 Subject: [PATCH 007/128] don't launch wasm event streamer if there are zero events --- mm2src/mm2_main/src/lp_native_dex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index bdc8079381..6d361888cc 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -448,7 +448,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { #[cfg(target_arch = "wasm32")] fn init_wasm_event_streaming(ctx: &MmArc) { - if ctx.event_stream_configuration.is_some() { + if ctx.event_stream_configuration.total_active_events() != 0 { ctx.spawner().spawn(handle_worker_stream(ctx.clone())); } } From eaca49bd232cc74c1905019ca2a7ffb4a8caf540 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 8 Jul 2024 16:21:00 +0200 Subject: [PATCH 008/128] don't error if in native mode while balance streaming enabled --- mm2src/coins/utxo/utxo_balance_events.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 835e0b9f03..4abc988d1b 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -91,10 +91,9 @@ impl EventBehaviour for UtxoBalanceEventStreamer { } if coin.as_ref().rpc_client.is_native() { - let e = "Native mode is not supported for event streaming"; - tx.send(EventInitStatus::Failed(e.to_string())) - .expect(RECEIVER_DROPPED_MSG); - panic!("{}", e); + // We won't consider this an error but just an unsupported scenario and continue running. + tx.send(EventInitStatus::Success).expect(RECEIVER_DROPPED_MSG); + panic!("Native mode is not supported for event streaming", e); } let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { From b8934837e708b15557428889a4b206c484c7e89b Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 8 Jul 2024 20:07:08 +0200 Subject: [PATCH 009/128] generalize event streaming config to be any json config --- mm2src/mm2_event_stream/Cargo.toml | 1 + mm2src/mm2_event_stream/src/behaviour.rs | 16 ++----- mm2src/mm2_event_stream/src/lib.rs | 19 +++----- mm2src/mm2_main/src/heartbeat_event.rs | 59 +++++++++++++++--------- mm2src/mm2_main/src/lp_native_dex.rs | 29 ++++++------ mm2src/mm2_net/src/network_event.rs | 59 +++++++++++++++--------- 6 files changed, 99 insertions(+), 84 deletions(-) diff --git a/mm2src/mm2_event_stream/Cargo.toml b/mm2src/mm2_event_stream/Cargo.toml index adf20e7ee2..369a22197d 100644 --- a/mm2src/mm2_event_stream/Cargo.toml +++ b/mm2src/mm2_event_stream/Cargo.toml @@ -10,6 +10,7 @@ common = { path = "../common" } futures = { version = "0.3", default-features = false } parking_lot = "0.12" serde = { version = "1", features = ["derive", "rc"] } +serde_json = { version = "1", features = ["preserve_order", "raw_value"] } tokio = { version = "1", features = ["sync"] } [dev-dependencies] diff --git a/mm2src/mm2_event_stream/src/behaviour.rs b/mm2src/mm2_event_stream/src/behaviour.rs index ff2cfbefa9..0d7786d39a 100644 --- a/mm2src/mm2_event_stream/src/behaviour.rs +++ b/mm2src/mm2_event_stream/src/behaviour.rs @@ -2,13 +2,6 @@ use crate::{ErrorEventName, EventName, EventStreamConfiguration}; use async_trait::async_trait; use futures::channel::oneshot; -#[derive(Clone, Debug)] -pub enum EventInitStatus { - Inactive, - Success, - Failed(String), -} - #[async_trait] pub trait EventBehaviour { /// Returns the unique name of the event as an EventName enum variant. @@ -19,9 +12,10 @@ pub trait EventBehaviour { fn error_event_name() -> ErrorEventName { ErrorEventName::GenericError } /// Event handler that is responsible for broadcasting event data to the streaming channels. - async fn handle(self, interval: f64, tx: oneshot::Sender); + /// + /// `tx` is a oneshot sender that is used to send the initialization status of the event. + async fn handle(self, tx: oneshot::Sender>); - /// Spawns the `Self::handle` in a separate thread if the event is active according to the mm2 configuration. - /// Does nothing if the event is not active. - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus; + /// Spawns the `Self::handle` in a separate thread. + async fn spawn(self) -> Result<(), String>; } diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 1dc15bcd53..203a1815cb 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,6 +1,9 @@ -use serde::Deserialize; use std::collections::HashMap; use std::fmt; + +use serde::Deserialize; +use serde_json::Value as Json; + #[cfg(target_arch = "wasm32")] use std::path::PathBuf; #[cfg(target_arch = "wasm32")] @@ -77,7 +80,7 @@ pub struct EventStreamConfiguration { #[serde(default)] pub access_control_allow_origin: String, #[serde(default)] - active_events: HashMap, + active_events: HashMap, /// The path to the worker script for event streaming. #[cfg(target_arch = "wasm32")] #[serde(default = "default_worker_path")] @@ -88,16 +91,6 @@ pub struct EventStreamConfiguration { #[inline] fn default_worker_path() -> PathBuf { PathBuf::from(DEFAULT_WORKER_PATH) } -/// Represents the configuration for a specific event within the event stream. -#[derive(Clone, Default, Deserialize)] -pub struct EventConfig { - /// The interval in seconds at which the event should be streamed. - #[serde(default = "default_stream_interval")] - pub stream_interval_seconds: f64, -} - -const fn default_stream_interval() -> f64 { 5. } - impl Default for EventStreamConfiguration { fn default() -> Self { Self { @@ -112,7 +105,7 @@ impl Default for EventStreamConfiguration { impl EventStreamConfiguration { /// Retrieves the configuration for a specific event by its name. #[inline] - pub fn get_event(&self, event_name: &EventName) -> Option { + pub fn get_event(&self, event_name: &EventName) -> Option { self.active_events.get(event_name).cloned() } diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 6c4d19d77b..02cdc20fb3 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -3,23 +3,38 @@ use common::{executor::{SpawnFuture, Timer}, log::info}; use futures::channel::oneshot::{self, Receiver, Sender}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::{EventBehaviour, EventInitStatus}, - Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{behaviour::EventBehaviour, Event, EventName, EventStreamConfiguration}; +use serde::Deserialize; +use serde_json::Value as Json; + +#[derive(Debug, Deserialize)] +struct HeartbeatEventConfig { + #[serde(default = "default_stream_interval")] + pub stream_interval_seconds: f64, +} + +const fn default_stream_interval() -> f64 { 5. } pub struct HeartbeatEvent { + config: HeartbeatEventConfig, ctx: MmArc, } impl HeartbeatEvent { - pub fn new(ctx: MmArc) -> Self { Self { ctx } } + pub fn try_new(config: Json, ctx: MmArc) -> Result { + Ok(Self { + config: serde_json::from_value(config)?, + ctx, + }) + } } #[async_trait] impl EventBehaviour for HeartbeatEvent { fn event_name() -> EventName { EventName::HEARTBEAT } - async fn handle(self, interval: f64, tx: oneshot::Sender) { - tx.send(EventInitStatus::Success).unwrap(); + async fn handle(self, tx: oneshot::Sender>) { + tx.send(Ok(())).unwrap(); loop { self.ctx @@ -27,26 +42,24 @@ impl EventBehaviour for HeartbeatEvent { .broadcast(Event::new(Self::event_name().to_string(), json!({}).to_string())) .await; - Timer::sleep(interval).await; + Timer::sleep(self.config.stream_interval_seconds).await; } } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - info!( - "{} event is activated with {} seconds interval.", - Self::event_name(), - event.stream_interval_seconds - ); - - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - self.ctx.spawner().spawn(self.handle(event.stream_interval_seconds, tx)); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive - } + async fn spawn(self) -> Result<(), String> { + info!( + "{} event is activated with config: {:?}", + Self::event_name(), + self.config + ); + + let (tx, rx) = oneshot::channel(); + self.ctx.spawner().spawn(self.handle(tx)); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler dropped before sending an initialization status: {e}", + )) + }) } } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 6d361888cc..ac25480366 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -205,10 +205,8 @@ pub enum MmInitError { OrdersKickStartError(String), #[display(fmt = "Error initializing wallet: {}", _0)] WalletInitError(String), - #[display(fmt = "NETWORK event initialization failed: {}", _0)] - NetworkEventInitFailed(String), - #[display(fmt = "HEARTBEAT event initialization failed: {}", _0)] - HeartbeatEventInitFailed(String), + #[display(fmt = "Event streamer initialization failed: {}", _0)] + EventStreamerInitFailed(String), #[from_trait(WithHwRpcError::hw_rpc_error)] #[display(fmt = "{}", _0)] HwError(HwRpcError), @@ -428,19 +426,20 @@ fn migrate_db(ctx: &MmArc) -> MmInitResult<()> { fn migration_1(_ctx: &MmArc) {} async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { - // This condition only executed if events were enabled in mm2 configuration. - if let EventInitStatus::Failed(err) = NetworkEvent::new(ctx.clone()) - .spawn_if_active(&ctx.event_stream_configuration) - .await - { - return MmError::err(MmInitError::NetworkEventInitFailed(err)); + if let Some(config) = ctx.event_stream_configuration.get_event(&NetworkEvent::event_name()) { + NetworkEvent::try_new(config, ctx.clone()) + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))? + .spawn() + .await + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e}")))?; } - if let EventInitStatus::Failed(err) = HeartbeatEvent::new(ctx.clone()) - .spawn_if_active(&ctx.event_stream_configuration) - .await - { - return MmError::err(MmInitError::HeartbeatEventInitFailed(err)); + if let Some(config) = ctx.event_stream_configuration.get_event(&HeartbeatEvent::event_name()) { + HeartbeatEvent::try_new(config, ctx.clone()) + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")))? + .spawn() + .await + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e}")))?; } Ok(()) diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index b88655f383..21c239fd7b 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -5,27 +5,42 @@ use common::{executor::{SpawnFuture, Timer}, use futures::channel::oneshot::{self, Receiver, Sender}; use mm2_core::mm_ctx::MmArc; pub use mm2_event_stream::behaviour::EventBehaviour; -use mm2_event_stream::{behaviour::EventInitStatus, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{Event, EventName, EventStreamConfiguration}; use mm2_libp2p::behaviours::atomicdex; -use serde_json::json; +use serde::Deserialize; +use serde_json::{json, Value as Json}; + +#[derive(Debug, Deserialize)] +struct NetworkEventConfig { + #[serde(default = "default_stream_interval")] + pub stream_interval_seconds: f64, +} + +const fn default_stream_interval() -> f64 { 5. } pub struct NetworkEvent { + config: NetworkEventConfig, ctx: MmArc, } impl NetworkEvent { - pub fn new(ctx: MmArc) -> Self { Self { ctx } } + pub fn try_new(config: Json, ctx: MmArc) -> Result { + Ok(Self { + config: serde_json::from_value(config)?, + ctx, + }) + } } #[async_trait] impl EventBehaviour for NetworkEvent { fn event_name() -> EventName { EventName::NETWORK } - async fn handle(self, interval: f64, tx: oneshot::Sender) { + async fn handle(self, tx: oneshot::Sender>) { let p2p_ctx = P2PContext::fetch_from_mm_arc(&self.ctx); let mut previously_sent = json!({}); - tx.send(EventInitStatus::Success).unwrap(); + tx.send(Ok(())).unwrap(); loop { let p2p_cmd_tx = p2p_ctx.cmd_tx.lock().clone(); @@ -53,25 +68,25 @@ impl EventBehaviour for NetworkEvent { previously_sent = event_data; } - Timer::sleep(interval).await; + Timer::sleep(self.config.stream_interval_seconds).await; } } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - info!( - "NETWORK event is activated with {} seconds interval.", - event.stream_interval_seconds - ); - - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - self.ctx.spawner().spawn(self.handle(event.stream_interval_seconds, tx)); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive - } + async fn spawn(self) -> Result<(), String> { + info!( + "{} event is activated with config: {:?}", + Self::event_name(), + self.config + ); + + let (tx, rx) = oneshot::channel(); + self.ctx.spawner().spawn(self.handle(tx)); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler dropped before sending an initialization status: {}", + e + )) + }) } } From 737de5b3a1eeec82f90d71456afd94d073a3f474 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 9 Jul 2024 21:53:49 +0200 Subject: [PATCH 010/128] rework the current events to work with arbitrary input config --- Cargo.lock | 1 + mm2src/coins/eth.rs | 15 ++- mm2src/coins/eth/eth_balance_events.rs | 105 +++++++++++++----- mm2src/coins/lp_coins.rs | 2 + mm2src/coins/streaming_events_config.rs | 22 ++++ mm2src/coins/tendermint/mod.rs | 2 +- .../tendermint/tendermint_balance_events.rs | 94 ++++++++++------ mm2src/coins/utxo/utxo_balance_events.rs | 98 +++++++++------- .../utxo/utxo_builder/utxo_arc_builder.rs | 23 +++- mm2src/coins/z_coin.rs | 15 ++- mm2src/coins/z_coin/z_balance_streaming.rs | 100 ++++++++++------- .../src/tendermint_with_assets_activation.rs | 24 +++- mm2src/mm2_core/src/mm_ctx.rs | 1 + mm2src/mm2_event_stream/src/behaviour.rs | 8 +- mm2src/mm2_event_stream/src/lib.rs | 4 +- mm2src/mm2_main/src/heartbeat_event.rs | 7 +- mm2src/mm2_main/src/lp_native_dex.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 7 +- mm2src/mm2_net/src/wasm_event_stream.rs | 2 +- 19 files changed, 349 insertions(+), 183 deletions(-) create mode 100644 mm2src/coins/streaming_events_config.rs diff --git a/Cargo.lock b/Cargo.lock index 755ad68862..3229bcbe56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4479,6 +4479,7 @@ dependencies = [ "futures 0.3.28", "parking_lot 0.12.0", "serde", + "serde_json", "tokio", "wasm-bindgen-test", ] diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 83e6e1d54c..185bc00b8d 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -63,6 +63,7 @@ use crypto::privkey::key_pair_from_secret; use crypto::{Bip44Chain, CryptoCtx, CryptoCtxError, GlobalHDAccountArc, KeyPairPolicy}; use derive_more::Display; use enum_derives::EnumFromStringify; +use eth_balance_events::EthBalanceEventStreamer; use ethabi::{Contract, Function, Token}; use ethcore_transaction::tx_builders::TxBuilderError; use ethcore_transaction::{Action, TransactionWrapper, TransactionWrapperBuilder as UnSignedEthTxBuilder, @@ -78,7 +79,7 @@ use http::Uri; use instant::Instant; use keys::Public as HtlcPubKey; use mm2_core::mm_ctx::{MmArc, MmWeak}; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; +use mm2_event_stream::behaviour::EventBehaviour; use mm2_net::transport::{GuiAuthValidation, GuiAuthValidationGenerator}; use mm2_number::bigdecimal_custom::CheckedDivision; use mm2_number::{BigDecimal, BigUint, MmNumber}; @@ -5282,12 +5283,16 @@ impl EthCoin { } async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - if let EventInitStatus::Failed(err) = - EventBehaviour::spawn_if_active(self.clone(), &ctx.event_stream_configuration).await + if let Some(config) = ctx + .event_stream_configuration + .get_event(&EthBalanceEventStreamer::event_name()) { - return ERR!("Failed spawning balance events. Error: {}", err); + EthBalanceEventStreamer::try_new(config, self.clone()) + .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))? + .spawn() + .await + .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {}", e))?; } - Ok(()) } diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 231aa68507..188bc3ab18 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -2,21 +2,54 @@ use async_trait::async_trait; use common::{executor::{AbortSettings, SpawnAbortable, Timer}, log, Future01CompatExt}; use ethereum_types::Address; -use futures::{channel::oneshot::{self, Receiver, Sender}, - stream::FuturesUnordered, - StreamExt}; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{behaviour::{EventBehaviour, EventInitStatus}, - ErrorEventName, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; use mm2_number::BigDecimal; +use serde::Deserialize; +use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; use super::EthCoin; +use crate::streaming_events_config::BalanceEventConfig; use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, BalanceError, CoinWithDerivationMethod, MmCoin}; +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SingleEthCoinConfig { + #[serde(default = "default_stream_interval")] + pub stream_interval_seconds: f64, +} + +const fn default_stream_interval() -> f64 { 10. } + +pub struct EthBalanceEventStreamer { + /// Whether the event is enabled for this coin. + enabled: bool, + /// The period in seconds between each balance check. + interval: f64, + coin: EthCoin, +} + +impl EthBalanceEventStreamer { + pub fn try_new(config: Json, coin: EthCoin) -> serde_json::Result { + let config: BalanceEventConfig = serde_json::from_value(config)?; + let coin_config: Option = match config.find_coin(&coin.ticker) { + // Try to parse the coin config. + Some(c) => Some(serde_json::from_value(c)?), + None => None, + }; + Ok(Self { + enabled: coin_config.is_some(), + interval: coin_config.map(|c| c.stream_interval_seconds).unwrap_or(0.0), + coin, + }) + } +} + struct BalanceData { ticker: String, address: String, @@ -113,12 +146,12 @@ async fn fetch_balance( } #[async_trait] -impl EventBehaviour for EthCoin { +impl EventBehaviour for EthBalanceEventStreamer { fn event_name() -> EventName { EventName::CoinBalance } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, interval: f64, tx: oneshot::Sender) { + async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; async fn start_polling(coin: EthCoin, ctx: MmArc, interval: f64) { @@ -147,7 +180,7 @@ impl EventBehaviour for EthCoin { let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); ctx.stream_channel_controller .broadcast(Event::new( - format!("{}:{}", EthCoin::error_event_name(), coin.ticker), + format!("{}:{}", EthBalanceEventStreamer::error_event_name(), coin.ticker), e.to_string(), )) .await; @@ -184,7 +217,12 @@ impl EventBehaviour for EthCoin { let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); ctx.stream_channel_controller .broadcast(Event::new( - format!("{}:{}:{}", EthCoin::error_event_name(), err.ticker, err.address), + format!( + "{}:{}:{}", + EthBalanceEventStreamer::error_event_name(), + err.ticker, + err.address + ), e.to_string(), )) .await; @@ -195,7 +233,7 @@ impl EventBehaviour for EthCoin { if !balance_updates.is_empty() { ctx.stream_channel_controller .broadcast(Event::new( - EthCoin::event_name().to_string(), + EthBalanceEventStreamer::event_name().to_string(), json!(balance_updates).to_string(), )) .await; @@ -205,36 +243,43 @@ impl EventBehaviour for EthCoin { } } - let ctx = match MmArc::from_weak(&self.ctx) { + let ctx = match MmArc::from_weak(&self.coin.ctx) { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(EventInitStatus::Failed(msg.to_owned())) - .expect(RECEIVER_DROPPED_MSG); + tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; - tx.send(EventInitStatus::Success).expect(RECEIVER_DROPPED_MSG); + tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - start_polling(self, ctx, interval).await + start_polling(self.coin, ctx, self.interval).await } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - log::info!("{} event is activated for {}", Self::event_name(), self.ticker,); - - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - let fut = self.clone().handle(event.stream_interval_seconds, tx); - let settings = - AbortSettings::info_on_abort(format!("{} event is stopped for {}.", Self::event_name(), self.ticker)); - self.spawner().spawn_with_settings(fut, settings); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive + async fn spawn(self) -> Result<(), String> { + if !self.enabled { + return Ok(()); } + log::info!( + "{} event is activated for {}, and polling every {}s", + Self::event_name(), + self.coin.ticker, + self.interval + ); + + let (tx, rx) = oneshot::channel(); + let settings = AbortSettings::info_on_abort(format!( + "{} event is stopped for {}.", + Self::event_name(), + self.coin.ticker + )); + self.coin.spawner().spawn_with_settings(self.handle(tx), settings); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + }) } } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 6d0e82c756..303b60d9f9 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -239,6 +239,8 @@ use coin_errors::{MyAddressError, ValidatePaymentError, ValidatePaymentFut, Vali #[cfg(test)] pub mod coins_tests; +mod streaming_events_config; + pub mod eth; use eth::GetValidEthWithdrawAddError; use eth::{eth_coin_from_conf_and_request, get_eth_address, EthCoin, EthGasDetailsErr, EthTxFeeDetails, diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs new file mode 100644 index 0000000000..f0883cbb65 --- /dev/null +++ b/mm2src/coins/streaming_events_config.rs @@ -0,0 +1,22 @@ +use serde::Deserialize; +use serde_json::Value as Json; +use std::collections::HashMap; + +#[derive(Debug, Deserialize)] +pub(crate) struct BalanceEventConfig { + coins: HashMap, +} + +impl BalanceEventConfig { + /// Returns the coin's configuration as a JSON object if it exists. + /// + /// Note that the `ticker` is looked up in a case-insensitive manner. + pub(crate) fn find_coin(&self, ticker: &str) -> Option { + // FIXME: Is this OK? are coin tickers like an identifier for the coin? Could two coins have the same ticker? + let ticker = ticker.to_lowercase(); + self.coins + .iter() + .find(|(key, _)| key.to_lowercase() == ticker) + .map(|(_, value)| value.clone()) + } +} diff --git a/mm2src/coins/tendermint/mod.rs b/mm2src/coins/tendermint/mod.rs index 78009b5db8..9f9a5b29a9 100644 --- a/mm2src/coins/tendermint/mod.rs +++ b/mm2src/coins/tendermint/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod ethermint_account; pub mod htlc; mod ibc; mod rpc; -mod tendermint_balance_events; +pub mod tendermint_balance_events; mod tendermint_coin; mod tendermint_token; pub mod tendermint_tx_history_v2; diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 1a55dfa768..db45d0d8ac 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -1,26 +1,53 @@ use async_trait::async_trait; use common::{executor::{AbortSettings, SpawnAbortable}, http_uri_to_ws_address, log}; -use futures::channel::oneshot::{self, Receiver, Sender}; +use futures::channel::oneshot; use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::{EventBehaviour, EventInitStatus}, - ErrorEventName, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; use mm2_number::BigDecimal; +use serde::Deserialize; +use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; use super::TendermintCoin; +use crate::streaming_events_config::BalanceEventConfig; use crate::{tendermint::TendermintCommons, utxo::utxo_common::big_decimal_from_sat_unsigned, MarketCoinOps, MmCoin}; +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct EmptyConfig {} + +pub struct TendermintBalanceEventStreamer { + /// Whether the event is enabled for this coin. + enabled: bool, + coin: TendermintCoin, +} + +impl TendermintBalanceEventStreamer { + pub fn try_new(config: Json, coin: TendermintCoin) -> serde_json::Result { + let config: BalanceEventConfig = serde_json::from_value(config)?; + let enabled = match config.find_coin(coin.ticker()) { + // This is just an extra check to make sure the config is correct (no config) + Some(c) => serde_json::from_value::(c).map(|_| true)?, + None => false, + }; + Ok(Self { enabled, coin }) + } +} + #[async_trait] -impl EventBehaviour for TendermintCoin { +impl EventBehaviour for TendermintBalanceEventStreamer { fn event_name() -> EventName { EventName::CoinBalance } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, _interval: f64, tx: oneshot::Sender) { + async fn handle(self, tx: oneshot::Sender>) { + const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let coin = self.coin; + fn generate_subscription_query(query_filter: String) -> String { let mut params = serde_json::Map::with_capacity(1); params.insert("query".to_owned(), RpcValue::String(query_filter)); @@ -35,17 +62,16 @@ impl EventBehaviour for TendermintCoin { serde_json::to_string(&q).expect("This should never happen") } - let ctx = match MmArc::from_weak(&self.ctx) { + let ctx = match MmArc::from_weak(&coin.ctx) { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(EventInitStatus::Failed(msg.to_owned())) - .expect("Receiver is dropped, which should never happen."); + tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; - let account_id = self.account_id.to_string(); + let account_id = coin.account_id.to_string(); let mut current_balances: HashMap = HashMap::new(); let receiver_q = generate_subscription_query(format!("coin_received.receiver = '{}'", account_id)); @@ -54,11 +80,10 @@ impl EventBehaviour for TendermintCoin { let spender_q = generate_subscription_query(format!("coin_spent.spender = '{}'", account_id)); let spender_q = tokio_tungstenite_wasm::Message::Text(spender_q); - tx.send(EventInitStatus::Success) - .expect("Receiver is dropped, which should never happen."); + tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); loop { - let node_uri = match self.rpc_client().await { + let node_uri = match coin.rpc_client().await { Ok(client) => client.uri(), Err(e) => { log::error!("{e}"); @@ -118,8 +143,8 @@ impl EventBehaviour for TendermintCoin { let mut balance_updates = vec![]; for denom in denoms { - if let Some((ticker, decimals)) = self.active_ticker_and_decimals_from_denom(&denom) { - let balance_denom = match self.account_balance_for_denom(&self.account_id, denom).await { + if let Some((ticker, decimals)) = coin.active_ticker_and_decimals_from_denom(&denom) { + let balance_denom = match coin.account_balance_for_denom(&coin.account_id, denom).await { Ok(balance_denom) => balance_denom, Err(e) => { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); @@ -171,29 +196,24 @@ impl EventBehaviour for TendermintCoin { } } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - log::info!( - "{} event is activated for {}. `stream_interval_seconds`({}) has no effect on this.", - Self::event_name(), - self.ticker(), - event.stream_interval_seconds - ); - - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - let fut = self.clone().handle(event.stream_interval_seconds, tx); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.ticker() - )); - self.spawner().spawn_with_settings(fut, settings); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive + async fn spawn(self) -> Result<(), String> { + if !self.enabled { + return Ok(()); } + log::info!("{} event is activated for {}.", Self::event_name(), self.coin.ticker(),); + + let (tx, rx) = oneshot::channel(); + let settings = AbortSettings::info_on_abort(format!( + "{} event is stopped for {}.", + Self::event_name(), + self.coin.ticker() + )); + self.coin.spawner().spawn_with_settings(self.handle(tx), settings); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + }) } } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 4abc988d1b..df7ae571ed 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -1,15 +1,17 @@ use async_trait::async_trait; use common::{executor::{AbortSettings, SpawnAbortable, Timer}, log, Future01CompatExt}; -use futures::channel::oneshot::{self, Receiver, Sender}; +use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::{EventBehaviour, EventInitStatus}, - ErrorEventName, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; +use serde::Deserialize; +use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; +use crate::streaming_events_config::BalanceEventConfig; use crate::{utxo::{output_script, rpc_clients::electrum_script_hash, utxo_common::{address_balance, address_to_scripthash}, @@ -28,17 +30,30 @@ macro_rules! try_or_continue { }; } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct EmptyConfig {} + pub struct UtxoBalanceEventStreamer { + /// Whether the event is enabled for this coin. + enabled: bool, coin: UtxoStandardCoin, } impl UtxoBalanceEventStreamer { - pub fn new(utxo_arc: UtxoArc) -> Self { - Self { + pub fn try_new(config: Json, utxo_arc: UtxoArc) -> serde_json::Result { + let config: BalanceEventConfig = serde_json::from_value(config)?; + let enabled = match config.find_coin(&utxo_arc.conf.ticker) { + // This is just an extra check to make sure the config is correct (no config) + Some(c) => serde_json::from_value::(c).map(|_| true)?, + None => false, + }; + Ok(Self { + enabled, // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. // The UtxoArc might belong to a different coin type though. coin: UtxoStandardCoin::from(utxo_arc), - } + }) } } @@ -48,8 +63,7 @@ impl EventBehaviour for UtxoBalanceEventStreamer { fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - // FIXME: Move `interval` to `self`. - async fn handle(self, _interval: f64, tx: oneshot::Sender) { + async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; @@ -90,18 +104,11 @@ impl EventBehaviour for UtxoBalanceEventStreamer { Ok(scripthash_to_address_map) } - if coin.as_ref().rpc_client.is_native() { - // We won't consider this an error but just an unsupported scenario and continue running. - tx.send(EventInitStatus::Success).expect(RECEIVER_DROPPED_MSG); - panic!("Native mode is not supported for event streaming", e); - } - let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(EventInitStatus::Failed(msg.to_owned())) - .expect(RECEIVER_DROPPED_MSG); + tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; @@ -110,13 +117,12 @@ impl EventBehaviour for UtxoBalanceEventStreamer { Some(t) => t, None => { let e = "Scripthash notification receiver can not be empty."; - tx.send(EventInitStatus::Failed(e.to_string())) - .expect(RECEIVER_DROPPED_MSG); + tx.send(Err(e.to_string())).expect(RECEIVER_DROPPED_MSG); panic!("{}", e); }, }; - tx.send(EventInitStatus::Success).expect(RECEIVER_DROPPED_MSG); + tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); let mut scripthash_to_address_map = BTreeMap::default(); while let Some(message) = scripthash_notification_handler.lock().await.next().await { @@ -228,30 +234,38 @@ impl EventBehaviour for UtxoBalanceEventStreamer { } } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - log::info!( - "{} event is activated for {}. `stream_interval_seconds`({}) has no effect on this.", - Self::event_name(), - self.coin.ticker(), - event.stream_interval_seconds - ); + async fn spawn(self) -> Result<(), String> { + if !self.enabled { + return Ok(()); + } - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.coin.ticker() - )); - self.coin - .spawner() - .spawn_with_settings(self.handle(event.stream_interval_seconds, tx), settings); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive + if self.coin.as_ref().rpc_client.is_native() { + log::warn!( + "Native RPC client is not supported for {} event. Skipping event initialization.", + self.event_id(), + ); + // We won't consider this an error but just an unsupported scenario and continue running. + return Ok(()); } + + log::info!( + "{} event is activated for coin: {}", + Self::event_name(), + self.coin.ticker(), + ); + + let (tx, rx) = oneshot::channel(); + let settings = AbortSettings::info_on_abort(format!( + "{} event is stopped for {}.", + Self::event_name(), + self.coin.ticker() + )); + self.coin.spawner().spawn_with_settings(self.handle(tx), settings); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + }) } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 504219677f..97cd28a42a 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -14,7 +14,7 @@ use common::log::{debug, error, info, warn}; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; +use mm2_event_stream::behaviour::EventBehaviour; #[cfg(test)] use mocktopus::macros::*; use rand::Rng; use script::Builder; @@ -120,11 +120,24 @@ where spawn_block_header_utxo_loop(self.ticker, &utxo_arc, sync_handle, spv_conf); } - if let EventInitStatus::Failed(err) = UtxoBalanceEventStreamer::new(utxo_arc) - .spawn_if_active(&self.ctx().event_stream_configuration) - .await + if let Some(config) = self + .ctx() + .event_stream_configuration + .get_event(&UtxoBalanceEventStreamer::event_name()) { - return MmError::err(UtxoCoinBuildError::FailedSpawningBalanceEvents(err)); + UtxoBalanceEventStreamer::try_new(config, utxo_arc) + .map_to_mm(|e| { + UtxoCoinBuildError::FailedSpawningBalanceEvents(format!( + "Failed to initialize utxo event streaming: {e}" + )) + })? + .spawn() + .await + .map_to_mm(|e| { + UtxoCoinBuildError::FailedSpawningBalanceEvents(format!( + "Failed to spawn utxo event streaming: {e}" + )) + })?; } Ok(result_coin) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 3f1c5c7265..cfba2206ce 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -25,6 +25,7 @@ use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxDa use crate::utxo::{UnsupportedAddr, UtxoFeeDetails}; use crate::z_coin::storage::{BlockDbImpl, WalletDbShared}; use crate::z_coin::z_balance_streaming::ZBalanceEventHandler; +use crate::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; use crate::z_coin::z_tx_history::{fetch_tx_history_from_db, ZCoinTxHistoryItem}; use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, @@ -59,7 +60,7 @@ use keys::hash::H256; use keys::{KeyPair, Message, Public}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; +use mm2_event_stream::behaviour::EventBehaviour; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; use primitives::bytes::Bytes; @@ -661,11 +662,15 @@ impl ZCoin { } async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - let coin = self.clone(); - if let EventInitStatus::Failed(err) = - EventBehaviour::spawn_if_active(coin, &ctx.event_stream_configuration).await + if let Some(config) = ctx + .event_stream_configuration + .get_event(&ZCoinBalanceEventStreamer::event_name()) { - return ERR!("Failed spawning zcoin balance event with error: {}", err); + ZCoinBalanceEventStreamer::try_new(config, self.clone()) + .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))? + .spawn() + .await + .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {}", e))?; } Ok(()) diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 5f6d3e590a..a0db4dd558 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -1,4 +1,5 @@ use crate::common::Future01CompatExt; +use crate::streaming_events_config::BalanceEventConfig; use crate::z_coin::ZCoin; use crate::{MarketCoinOps, MmCoin}; @@ -7,34 +8,56 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::log::{error, info}; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; -use futures::channel::oneshot::{Receiver, Sender}; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; -use mm2_event_stream::{ErrorEventName, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::{ErrorEventName, Event, EventName}; +use serde::Deserialize; +use serde_json::Value as Json; use std::sync::Arc; pub type ZBalanceEventSender = UnboundedSender<()>; pub type ZBalanceEventHandler = Arc>>; +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct EmptyConfig {} + +pub struct ZCoinBalanceEventStreamer { + /// Whether the event is enabled for this coin. + enabled: bool, + coin: ZCoin, +} + +impl ZCoinBalanceEventStreamer { + pub fn try_new(config: Json, coin: ZCoin) -> serde_json::Result { + let config: BalanceEventConfig = serde_json::from_value(config)?; + let enabled = match config.find_coin(coin.ticker()) { + // This is just an extra check to make sure the config is correct (no config) + Some(c) => serde_json::from_value::(c).map(|_| true)?, + None => false, + }; + Ok(Self { enabled, coin }) + } +} + #[async_trait] -impl EventBehaviour for ZCoin { +impl EventBehaviour for ZCoinBalanceEventStreamer { fn event_name() -> EventName { EventName::CoinBalance } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, _interval: f64, tx: Sender) { + async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let coin = self.coin; macro_rules! send_status_on_err { ($match: expr, $sender: tt, $msg: literal) => { match $match { Some(t) => t, None => { - $sender - .send(EventInitStatus::Failed($msg.to_owned())) - .expect(RECEIVER_DROPPED_MSG); + $sender.send(Err($msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", $msg); }, } @@ -42,26 +65,26 @@ impl EventBehaviour for ZCoin { } let ctx = send_status_on_err!( - MmArc::from_weak(&self.as_ref().ctx), + MmArc::from_weak(&coin.as_ref().ctx), tx, "MM context must have been initialized already." ); let z_balance_change_handler = send_status_on_err!( - self.z_fields.z_balance_event_handler.as_ref(), + coin.z_fields.z_balance_event_handler.as_ref(), tx, "Z balance change receiver can not be empty." ); - tx.send(EventInitStatus::Success).expect(RECEIVER_DROPPED_MSG); + tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); // Locks the balance change handler, iterates through received events, and updates balance changes accordingly. let mut bal = z_balance_change_handler.lock().await; while (bal.next().await).is_some() { - match self.my_balance().compat().await { + match coin.my_balance().compat().await { Ok(balance) => { let payload = json!({ - "ticker": self.ticker(), - "address": self.my_z_address_encoded(), + "ticker": coin.ticker(), + "address": coin.my_z_address_encoded(), "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); @@ -70,7 +93,7 @@ impl EventBehaviour for ZCoin { .await; }, Err(err) => { - let ticker = self.ticker(); + let ticker = coin.ticker(); error!("Failed getting balance for '{ticker}'. Error: {err}"); let e = serde_json::to_value(err).expect("Serialization should't fail."); return ctx @@ -85,30 +108,29 @@ impl EventBehaviour for ZCoin { } } - async fn spawn_if_active(self, config: &EventStreamConfiguration) -> EventInitStatus { - if let Some(event) = config.get_event(&Self::event_name()) { - info!( - "{} event is activated for {} address {}. `stream_interval_seconds`({}) has no effect on this.", - Self::event_name(), - self.ticker(), - self.my_z_address_encoded(), - event.stream_interval_seconds - ); - - let (tx, rx): (Sender, Receiver) = oneshot::channel(); - let fut = self.clone().handle(event.stream_interval_seconds, tx); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.ticker() - )); - self.spawner().spawn_with_settings(fut, settings); - - rx.await.unwrap_or_else(|e| { - EventInitStatus::Failed(format!("Event initialization status must be received: {}", e)) - }) - } else { - EventInitStatus::Inactive + async fn spawn(self) -> Result<(), String> { + if !self.enabled { + return Ok(()); } + info!( + "{} event is activated for {} address {}.", + Self::event_name(), + self.coin.ticker(), + self.coin.my_z_address_encoded(), + ); + + let (tx, rx) = oneshot::channel(); + let settings = AbortSettings::info_on_abort(format!( + "{} event is stopped for {}.", + Self::event_name(), + self.coin.ticker() + )); + self.coin.spawner().spawn_with_settings(self.handle(tx), settings); + + rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + }) } } diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 5a79bbeb6e..0997322246 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -10,6 +10,7 @@ use crate::prelude::*; use async_trait::async_trait; use coins::hd_wallet::HDPathAccountToAddressId; use coins::my_tx_history_v2::TxHistoryStorage; +use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; use coins::tendermint::tendermint_tx_history_v2::tendermint_history_loop; use coins::tendermint::{tendermint_priv_key_policy, TendermintActivationPolicy, TendermintCoin, TendermintCommons, TendermintConf, TendermintInitError, TendermintInitErrorKind, TendermintProtocolInfo, @@ -20,7 +21,7 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; +use mm2_event_stream::behaviour::EventBehaviour; use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; @@ -372,11 +373,22 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { &self, config: &EventStreamConfiguration, ) -> Result<(), MmError> { - if let EventInitStatus::Failed(err) = EventBehaviour::spawn_if_active(self.clone(), config).await { - return MmError::err(TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::BalanceStreamInitError(err), - }); + if let Some(config) = config.get_event(&TendermintBalanceEventStreamer::event_name()) { + TendermintBalanceEventStreamer::try_new(config, self.clone()) + .map_to_mm(|e| TendermintInitError { + ticker: self.ticker().to_owned(), + kind: TendermintInitErrorKind::BalanceStreamInitError(format!( + "Failed to initialize tendermint balance streaming: {e}" + )), + })? + .spawn() + .await + .map_to_mm(|e| TendermintInitError { + ticker: self.ticker().to_owned(), + kind: TendermintInitErrorKind::BalanceStreamInitError(format!( + "Failed to spawn tendermint balance streaming handler: {e}" + )), + })?; } Ok(()) } diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index f22d0d12b2..6742f643cb 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -757,6 +757,7 @@ impl MmCtxBuilder { let event_stream_configuration = &ctx.conf["event_stream_configuration"]; if !event_stream_configuration.is_null() { + // FIXME: Consider validating all the fields in the event streaming configuration here. ctx.event_stream_configuration = json::from_value(event_stream_configuration.clone()) .expect("Invalid json value in 'event_stream_configuration'."); } diff --git a/mm2src/mm2_event_stream/src/behaviour.rs b/mm2src/mm2_event_stream/src/behaviour.rs index 0d7786d39a..bdbc75370d 100644 --- a/mm2src/mm2_event_stream/src/behaviour.rs +++ b/mm2src/mm2_event_stream/src/behaviour.rs @@ -1,12 +1,16 @@ -use crate::{ErrorEventName, EventName, EventStreamConfiguration}; +use crate::{ErrorEventName, EventName}; use async_trait::async_trait; use futures::channel::oneshot; #[async_trait] pub trait EventBehaviour { - /// Returns the unique name of the event as an EventName enum variant. + /// Returns the name of the event as an EventName enum variant. fn event_name() -> EventName; + /// Returns a human readable unique identifier for the event streamer. + /// No other event should have the same identifier. + fn event_id(&self) -> String { unimplemented!() } + /// Returns the name of the error event as an ErrorEventName enum variant. /// By default, it returns `ErrorEventName::GenericError,` which shows as "ERROR" in the event stream. fn error_event_name() -> ErrorEventName { ErrorEventName::GenericError } diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 203a1815cb..9402e8d237 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -105,9 +105,7 @@ impl Default for EventStreamConfiguration { impl EventStreamConfiguration { /// Retrieves the configuration for a specific event by its name. #[inline] - pub fn get_event(&self, event_name: &EventName) -> Option { - self.active_events.get(event_name).cloned() - } + pub fn get_event(&self, event_name: &EventName) -> Option { self.active_events.get(event_name).cloned() } /// Gets the total number of active events in the configuration. #[inline] diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 02cdc20fb3..d6981ef9f0 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -1,13 +1,14 @@ use async_trait::async_trait; use common::{executor::{SpawnFuture, Timer}, log::info}; -use futures::channel::oneshot::{self, Receiver, Sender}; +use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::EventBehaviour, Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{behaviour::EventBehaviour, Event, EventName}; use serde::Deserialize; use serde_json::Value as Json; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] struct HeartbeatEventConfig { #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, @@ -23,7 +24,7 @@ pub struct HeartbeatEvent { impl HeartbeatEvent { pub fn try_new(config: Json, ctx: MmArc) -> Result { Ok(Self { - config: serde_json::from_value(config)?, + config: serde_json::from_value(config).map_err(|e| e.to_string())?, ctx, }) } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index ac25480366..84c78e6029 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -28,7 +28,7 @@ use enum_derives::EnumFromTrait; use mm2_core::mm_ctx::{MmArc, MmCtx}; use mm2_err_handle::common_errors::InternalError; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::{EventBehaviour, EventInitStatus}; +use mm2_event_stream::behaviour::EventBehaviour; use mm2_libp2p::behaviours::atomicdex::{GossipsubConfig, DEPRECATED_NETID_LIST}; use mm2_libp2p::{spawn_gossipsub, AdexBehaviourError, NodeType, RelayAddress, RelayAddressError, SeedNodeInfo, SwarmRuntime, WssCerts}; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 21c239fd7b..53383ab2c2 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -2,15 +2,16 @@ use crate::p2p::P2PContext; use async_trait::async_trait; use common::{executor::{SpawnFuture, Timer}, log::info}; -use futures::channel::oneshot::{self, Receiver, Sender}; +use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; pub use mm2_event_stream::behaviour::EventBehaviour; -use mm2_event_stream::{Event, EventName, EventStreamConfiguration}; +use mm2_event_stream::{Event, EventName}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; use serde_json::{json, Value as Json}; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] struct NetworkEventConfig { #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, @@ -26,7 +27,7 @@ pub struct NetworkEvent { impl NetworkEvent { pub fn try_new(config: Json, ctx: MmArc) -> Result { Ok(Self { - config: serde_json::from_value(config)?, + config: serde_json::from_value(config).map_err(|e| e.to_string())?, ctx, }) } diff --git a/mm2src/mm2_net/src/wasm_event_stream.rs b/mm2src/mm2_net/src/wasm_event_stream.rs index f3e381cd3d..a92134d6bc 100644 --- a/mm2src/mm2_net/src/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/wasm_event_stream.rs @@ -14,7 +14,7 @@ unsafe impl Send for SendableMessagePort {} pub async fn handle_worker_stream(ctx: MmArc) { let config = &ctx.event_stream_configuration; - let mut channel_controller = ctx.stream_channel_controller.clone(); + let channel_controller = ctx.stream_channel_controller.clone(); let mut rx = channel_controller.create_channel(config.total_active_events()); let worker_path = config From 929279f6ba0cec65b0015364ced4a23788d1c5aa Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 10 Jul 2024 14:04:59 +0200 Subject: [PATCH 011/128] add always_send parameter to network event config --- mm2src/mm2_net/src/network_event.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 53383ab2c2..d71738f89e 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -15,6 +15,9 @@ use serde_json::{json, Value as Json}; struct NetworkEventConfig { #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, + /// Always (force) send the event data, even if it's the same as the previous one. + #[serde(default)] + pub always_send: bool, } const fn default_stream_interval() -> f64 { 5. } @@ -60,7 +63,7 @@ impl EventBehaviour for NetworkEvent { "relay_mesh": relay_mesh, }); - if previously_sent != event_data { + if previously_sent != event_data || self.config.always_send { self.ctx .stream_channel_controller .broadcast(Event::new(Self::event_name().to_string(), event_data.to_string())) From 4456c746ca6b8ad53114b5c876bf9f6e07c83527 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 10 Jul 2024 14:06:37 +0200 Subject: [PATCH 012/128] balance is a better naming, no need for serde_rename now --- mm2src/coins/eth/eth_balance_events.rs | 2 +- mm2src/coins/tendermint/tendermint_balance_events.rs | 2 +- mm2src/coins/utxo/utxo_balance_events.rs | 2 +- mm2src/coins/z_coin/z_balance_streaming.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 5 ++--- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 188bc3ab18..a9a0be686e 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -147,7 +147,7 @@ async fn fetch_balance( #[async_trait] impl EventBehaviour for EthBalanceEventStreamer { - fn event_name() -> EventName { EventName::CoinBalance } + fn event_name() -> EventName { EventName::BALANCE } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index db45d0d8ac..5a4be0f797 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -40,7 +40,7 @@ impl TendermintBalanceEventStreamer { #[async_trait] impl EventBehaviour for TendermintBalanceEventStreamer { - fn event_name() -> EventName { EventName::CoinBalance } + fn event_name() -> EventName { EventName::BALANCE } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index df7ae571ed..10d60d0b76 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -59,7 +59,7 @@ impl UtxoBalanceEventStreamer { #[async_trait] impl EventBehaviour for UtxoBalanceEventStreamer { - fn event_name() -> EventName { EventName::CoinBalance } + fn event_name() -> EventName { EventName::BALANCE } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index a0db4dd558..e79ab3bff9 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -44,7 +44,7 @@ impl ZCoinBalanceEventStreamer { #[async_trait] impl EventBehaviour for ZCoinBalanceEventStreamer { - fn event_name() -> EventName { EventName::CoinBalance } + fn event_name() -> EventName { EventName::BALANCE } fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 9402e8d237..647c8645d7 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -38,8 +38,7 @@ impl Event { #[derive(Deserialize, Eq, Hash, PartialEq)] pub enum EventName { /// Indicates a change in the balance of a coin. - #[serde(rename = "COIN_BALANCE")] - CoinBalance, + BALANCE, /// Event triggered at regular intervals to indicate that the system is operational. HEARTBEAT, /// Returns p2p network information at a regular interval. @@ -49,7 +48,7 @@ pub enum EventName { impl fmt::Display for EventName { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::CoinBalance => write!(f, "COIN_BALANCE"), + Self::BALANCE => write!(f, "COIN_BALANCE"), Self::HEARTBEAT => write!(f, "HEARTBEAT"), Self::NETWORK => write!(f, "NETWORK"), } From 1b9aaa5add90d5b0a6ed47beeb69b7ecdc1cb038 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 10 Jul 2024 16:30:56 +0200 Subject: [PATCH 013/128] use a common empty struct for deser-ing empty config --- mm2src/coins/eth/eth_balance_events.rs | 1 + mm2src/coins/streaming_events_config.rs | 11 +++++++++++ mm2src/coins/tendermint/tendermint_balance_events.rs | 9 ++------- mm2src/coins/utxo/utxo_balance_events.rs | 9 ++------- mm2src/coins/z_coin/z_balance_streaming.rs | 9 ++------- mm2src/mm2_main/src/heartbeat_event.rs | 1 + mm2src/mm2_net/src/network_event.rs | 3 ++- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index a9a0be686e..351ba572d1 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -20,6 +20,7 @@ use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct SingleEthCoinConfig { + /// The time in seconds to wait before re-polling the balance and streaming. #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, } diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs index f0883cbb65..f7d71710e0 100644 --- a/mm2src/coins/streaming_events_config.rs +++ b/mm2src/coins/streaming_events_config.rs @@ -4,6 +4,10 @@ use std::collections::HashMap; #[derive(Debug, Deserialize)] pub(crate) struct BalanceEventConfig { + /// A map of coin tickers to their balance streaming configuration. + /// + /// The configuration doesn't have a specific structure at this point, every coin + /// has its own configuration structure. coins: HashMap, } @@ -20,3 +24,10 @@ impl BalanceEventConfig { .map(|(_, value)| value.clone()) } } + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +/// An empty configuration struct. +/// +/// It's used mainly to make sure there is no config provided. +pub(crate) struct EmptySubConfig {} \ No newline at end of file diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 5a4be0f797..7dc674dbed 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -8,18 +8,13 @@ use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; use mm2_number::BigDecimal; -use serde::Deserialize; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; use super::TendermintCoin; -use crate::streaming_events_config::BalanceEventConfig; +use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::{tendermint::TendermintCommons, utxo::utxo_common::big_decimal_from_sat_unsigned, MarketCoinOps, MmCoin}; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct EmptyConfig {} - pub struct TendermintBalanceEventStreamer { /// Whether the event is enabled for this coin. enabled: bool, @@ -31,7 +26,7 @@ impl TendermintBalanceEventStreamer { let config: BalanceEventConfig = serde_json::from_value(config)?; let enabled = match config.find_coin(coin.ticker()) { // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, + Some(c) => serde_json::from_value::(c).map(|_| true)?, None => false, }; Ok(Self { enabled, coin }) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 10d60d0b76..6e1d15d82a 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -6,12 +6,11 @@ use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; -use serde::Deserialize; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; -use crate::streaming_events_config::BalanceEventConfig; +use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::{utxo::{output_script, rpc_clients::electrum_script_hash, utxo_common::{address_balance, address_to_scripthash}, @@ -30,10 +29,6 @@ macro_rules! try_or_continue { }; } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct EmptyConfig {} - pub struct UtxoBalanceEventStreamer { /// Whether the event is enabled for this coin. enabled: bool, @@ -45,7 +40,7 @@ impl UtxoBalanceEventStreamer { let config: BalanceEventConfig = serde_json::from_value(config)?; let enabled = match config.find_coin(&utxo_arc.conf.ticker) { // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, + Some(c) => serde_json::from_value::(c).map(|_| true)?, None => false, }; Ok(Self { diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index e79ab3bff9..e48aa4c5cd 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -1,5 +1,5 @@ use crate::common::Future01CompatExt; -use crate::streaming_events_config::BalanceEventConfig; +use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::z_coin::ZCoin; use crate::{MarketCoinOps, MmCoin}; @@ -13,17 +13,12 @@ use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; use mm2_event_stream::behaviour::EventBehaviour; use mm2_event_stream::{ErrorEventName, Event, EventName}; -use serde::Deserialize; use serde_json::Value as Json; use std::sync::Arc; pub type ZBalanceEventSender = UnboundedSender<()>; pub type ZBalanceEventHandler = Arc>>; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct EmptyConfig {} - pub struct ZCoinBalanceEventStreamer { /// Whether the event is enabled for this coin. enabled: bool, @@ -35,7 +30,7 @@ impl ZCoinBalanceEventStreamer { let config: BalanceEventConfig = serde_json::from_value(config)?; let enabled = match config.find_coin(coin.ticker()) { // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, + Some(c) => serde_json::from_value::(c).map(|_| true)?, None => false, }; Ok(Self { enabled, coin }) diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index d6981ef9f0..4a425ba5bd 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -10,6 +10,7 @@ use serde_json::Value as Json; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct HeartbeatEventConfig { + /// The time in seconds to wait before sending another ping event. #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, } diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index d71738f89e..8f5c771bdd 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -13,9 +13,10 @@ use serde_json::{json, Value as Json}; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct NetworkEventConfig { + /// The time in seconds to wait after sending network info before sending another one. #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, - /// Always (force) send the event data, even if it's the same as the previous one. + /// Always (force) send network info data, even if it's the same as the previous one sent. #[serde(default)] pub always_send: bool, } From 9504df4accb46aa0e69b903687598b778caa028f Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 10 Jul 2024 22:37:04 +0200 Subject: [PATCH 014/128] add smart filter object to events --- mm2src/coins/eth/eth_balance_events.rs | 13 ++-- mm2src/coins/streaming_events_config.rs | 2 +- .../tendermint/tendermint_balance_events.rs | 10 ++- mm2src/coins/utxo/utxo_balance_events.rs | 45 ++++++++++--- mm2src/coins/z_coin/z_balance_streaming.rs | 7 +-- mm2src/mm2_core/src/data_asker.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 63 ++++++++++++++++--- mm2src/mm2_main/src/heartbeat_event.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 2 +- mm2src/mm2_net/src/sse_handler.rs | 12 ++-- 10 files changed, 113 insertions(+), 45 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 351ba572d1..ea16c11b85 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -180,9 +180,10 @@ impl EventBehaviour for EthBalanceEventStreamer { log::error!("Failed getting addresses for {}. Error: {}", coin.ticker, e); let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!("{}:{}", EthBalanceEventStreamer::error_event_name(), coin.ticker), - e.to_string(), + e, + None, )) .await; sleep_remaining_time(interval, now).await; @@ -217,14 +218,15 @@ impl EventBehaviour for EthBalanceEventStreamer { ); let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!( "{}:{}:{}", EthBalanceEventStreamer::error_event_name(), err.ticker, err.address ), - e.to_string(), + e, + None, )) .await; }, @@ -235,7 +237,8 @@ impl EventBehaviour for EthBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::new( EthBalanceEventStreamer::event_name().to_string(), - json!(balance_updates).to_string(), + json!(balance_updates), + None, )) .await; } diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs index f7d71710e0..2d7eb93a44 100644 --- a/mm2src/coins/streaming_events_config.rs +++ b/mm2src/coins/streaming_events_config.rs @@ -30,4 +30,4 @@ impl BalanceEventConfig { /// An empty configuration struct. /// /// It's used mainly to make sure there is no config provided. -pub(crate) struct EmptySubConfig {} \ No newline at end of file +pub(crate) struct EmptySubConfig {} diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 7dc674dbed..af8eb52afe 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -145,9 +145,10 @@ impl EventBehaviour for TendermintBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!("{}:{}", Self::error_event_name(), ticker), - e.to_string(), + e, + None, )) .await; @@ -180,10 +181,7 @@ impl EventBehaviour for TendermintBalanceEventStreamer { if !balance_updates.is_empty() { ctx.stream_channel_controller - .broadcast(Event::new( - Self::event_name().to_string(), - json!(balance_updates).to_string(), - )) + .broadcast(Event::new(Self::event_name().to_string(), json!(balance_updates), None)) .await; } } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 6e1d15d82a..c0d89c78f4 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -5,9 +5,10 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; +use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName, Filter}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; @@ -61,6 +62,7 @@ impl EventBehaviour for UtxoBalanceEventStreamer { async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; + let filter = Arc::new(UtxoBalanceEventFilter::new(coin.ticker())); async fn subscribe_to_addresses( utxo: &UtxoCoinFields, @@ -130,9 +132,10 @@ impl EventBehaviour for UtxoBalanceEventStreamer { log::error!("{e}"); ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!("{}:{}", Self::error_event_name(), coin.ticker()), - json!({ "error": e }).to_string(), + json!({ "error": e }), + Some(filter.clone()), )) .await; }, @@ -148,9 +151,10 @@ impl EventBehaviour for UtxoBalanceEventStreamer { log::error!("{e}"); ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!("{}:{}", Self::error_event_name(), coin.ticker()), - json!({ "error": e }).to_string(), + json!({ "error": e }), + Some(filter.clone()), )) .await; }, @@ -204,9 +208,10 @@ impl EventBehaviour for UtxoBalanceEventStreamer { // FIXME: Note that such an event isn't SSE-ed to any client since no body is listening to it. ctx.stream_channel_controller - .broadcast(Event::new( + .broadcast(Event::err( format!("{}:{}", Self::error_event_name(), ticker), - e.to_string(), + e, + Some(filter.clone()), )) .await; @@ -223,7 +228,8 @@ impl EventBehaviour for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::new( Self::event_name().to_string(), - json!(vec![payload]).to_string(), + json!(vec![payload]), + Some(filter.clone()), )) .await; } @@ -264,3 +270,26 @@ impl EventBehaviour for UtxoBalanceEventStreamer { }) } } + +struct UtxoBalanceEventFilter { + /// The event name we are looking for to let this event pass. + event_name_match: String, +} + +impl UtxoBalanceEventFilter { + pub fn new(ticker: &str) -> Self { + Self { + // The client requested event must have our ticker in that format to pass through. + event_name_match: format!("{}_{}", UtxoBalanceEventStreamer::event_name(), ticker), + } + } +} + +impl Filter for UtxoBalanceEventFilter { + fn filter(&self, message: &Json, requested_events: &HashSet) -> Option { + if requested_events.is_empty() || requested_events.contains(&self.event_name_match) { + return Some(message.clone()); + } + None + } +} diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index e48aa4c5cd..256c325714 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -84,7 +84,7 @@ impl EventBehaviour for ZCoinBalanceEventStreamer { }); ctx.stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), payload.to_string())) + .broadcast(Event::err(Self::event_name().to_string(), payload, None)) .await; }, Err(err) => { @@ -93,10 +93,7 @@ impl EventBehaviour for ZCoinBalanceEventStreamer { let e = serde_json::to_value(err).expect("Serialization should't fail."); return ctx .stream_channel_controller - .broadcast(Event::new( - format!("{}:{}", Self::error_event_name(), ticker), - e.to_string(), - )) + .broadcast(Event::new(format!("{}:{}", Self::error_event_name(), ticker), e, None)) .await; }, }; diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index 5d01310630..da64d621ac 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -69,7 +69,7 @@ impl MmCtx { }); self.stream_channel_controller - .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input.to_string())) + .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input, None)) .await; match receiver.await { diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 647c8645d7..dd7babdc0c 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,5 +1,6 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt; +use std::sync::Arc; use serde::Deserialize; use serde_json::Value as Json; @@ -11,27 +12,69 @@ const DEFAULT_WORKER_PATH: &str = "event_streaming_worker.js"; /// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { - _type: String, - message: String, + /// The type of the event (balance, network, swap, etc...). + event_type: String, + /// The message to be sent to the client. + message: Json, + /// The filter object to be used to determine whether the event should be sent or not. + /// It could also alter the event content + /// + /// The filter is wrapped in an `Arc` since the event producer should use it as a singleton + /// by using the same filter over and over again with multiple events. + filter: Option>, } impl Event { /// Creates a new `Event` instance with the specified event type and message. #[inline] - pub fn new(event_type: String, message: String) -> Self { + pub fn new(event_type: String, message: Json, filter: Option>) -> Self { Self { - _type: event_type, + event_type, message, + filter, } } - /// Gets the event type. + /// Create a new error `Event` instance with the specified error event type and message. #[inline] - pub fn event_type(&self) -> &str { &self._type } + pub fn err(event_type: String, message: Json, filter: Option>) -> Self { + Self { + event_type: format!("ERROR_{event_type}"), + message, + filter, + } + } - /// Gets the event message. - #[inline] - pub fn message(&self) -> &str { &self.message } + /// Returns the event type and message to be sent or `None` if the event should not be sent. + /// + /// Uses the `requested_events` to determine whether the event should be sent or not. + /// If `requested_events` is empty, this doesn't mean the event won't be sent, this is + /// decided by the event's filtering mechanism. + /// + /// `requested_events` could also be used to alter the event content (e.g. remove certain fields) + pub fn get_data(&self, requested_events: &HashSet) -> Option<(String, Json)> { + self.filter.as_ref().map_or_else( + // If no filter is set, send the event as is. + || Some((self.event_type.clone(), self.message.clone())), + |filter| { + filter + .filter(&self.message, requested_events) + .map(|message| (self.event_type.clone(), message)) + }, + ) + } +} + +/// A trait that defines the filtering mechanism for events. +/// +/// Each event has a filter that determines whether the event should be send out +/// to the client or not based on the client's requested events. +pub trait Filter: Send + Sync { + /// Filters the event based on the requested events. + /// + /// Returns the (maybe altered) message to be sent or `None` if the event should not be sent. + /// `requested_events` is a set of the events that the client asked to subscribe to (e.g. `BALANCE:BTC`) + fn filter(&self, message: &Json, requested_events: &HashSet) -> Option; } /// Event types streamed to clients through channels like Server-Sent Events (SSE). diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 4a425ba5bd..7fbde103a8 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -41,7 +41,7 @@ impl EventBehaviour for HeartbeatEvent { loop { self.ctx .stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), json!({}).to_string())) + .broadcast(Event::new(Self::event_name().to_string(), json!({}), None)) .await; Timer::sleep(self.config.stream_interval_seconds).await; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 8f5c771bdd..0acff08d6e 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -67,7 +67,7 @@ impl EventBehaviour for NetworkEvent { if previously_sent != event_data || self.config.always_send { self.ctx .stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), event_data.to_string())) + .broadcast(Event::new(Self::event_name().to_string(), event_data.clone(), None)) .await; previously_sent = event_data; diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index 0b05f69870..4ec8852f80 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -1,7 +1,7 @@ use hyper::{body::Bytes, Body, Request, Response}; use mm2_core::mm_ctx::MmArc; use serde_json::json; -use std::convert::Infallible; +use std::{collections::HashSet, convert::Infallible}; pub const SSE_ENDPOINT: &str = "/event-stream"; @@ -25,7 +25,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result(Bytes::from(format!("data: {data} \n\n"))); From d113c6fa0b1ea7b701769ec897ff840b0da804c1 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 12 Jul 2024 12:12:41 +0200 Subject: [PATCH 015/128] get rid of `error_event_name` also does some housekeeping, file splitting and renaming --- mm2src/coins/eth.rs | 2 +- mm2src/coins/eth/eth_balance_events.rs | 8 +- .../tendermint/tendermint_balance_events.rs | 10 +- mm2src/coins/utxo/utxo_balance_events.rs | 10 +- .../utxo/utxo_builder/utxo_arc_builder.rs | 2 +- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_balance_streaming.rs | 10 +- .../src/tendermint_with_assets_activation.rs | 2 +- mm2src/mm2_core/src/mm_ctx.rs | 3 +- mm2src/mm2_event_stream/src/behaviour.rs | 6 +- mm2src/mm2_event_stream/src/configuration.rs | 50 ++++++ mm2src/mm2_event_stream/src/event.rs | 94 ++++++++++ mm2src/mm2_event_stream/src/lib.rs | 165 +----------------- mm2src/mm2_event_stream/src/manager.rs | 7 + mm2src/mm2_main/src/heartbeat_event.rs | 2 +- mm2src/mm2_main/src/lp_native_dex.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 2 +- 17 files changed, 183 insertions(+), 194 deletions(-) create mode 100644 mm2src/mm2_event_stream/src/configuration.rs create mode 100644 mm2src/mm2_event_stream/src/event.rs create mode 100644 mm2src/mm2_event_stream/src/manager.rs diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 185bc00b8d..61c2c5354a 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -79,7 +79,7 @@ use http::Uri; use instant::Instant; use keys::Public as HtlcPubKey; use mm2_core::mm_ctx::{MmArc, MmWeak}; -use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::EventBehaviour; use mm2_net::transport::{GuiAuthValidation, GuiAuthValidationGenerator}; use mm2_number::bigdecimal_custom::CheckedDivision; use mm2_number::{BigDecimal, BigUint, MmNumber}; diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index ea16c11b85..15285e604f 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -6,7 +6,7 @@ use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; +use mm2_event_stream::{Event, EventBehaviour, EventName}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -150,8 +150,6 @@ async fn fetch_balance( impl EventBehaviour for EthBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } - fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; @@ -181,7 +179,7 @@ impl EventBehaviour for EthBalanceEventStreamer { let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", EthBalanceEventStreamer::error_event_name(), coin.ticker), + format!("{}:{}", EthBalanceEventStreamer::event_name(), coin.ticker), e, None, )) @@ -221,7 +219,7 @@ impl EventBehaviour for EthBalanceEventStreamer { .broadcast(Event::err( format!( "{}:{}:{}", - EthBalanceEventStreamer::error_event_name(), + EthBalanceEventStreamer::event_name(), err.ticker, err.address ), diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index af8eb52afe..0f6f9fe7dd 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -6,7 +6,7 @@ use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName}; +use mm2_event_stream::{Event, EventBehaviour, EventName}; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; @@ -37,8 +37,6 @@ impl TendermintBalanceEventStreamer { impl EventBehaviour for TendermintBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } - fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; @@ -145,11 +143,7 @@ impl EventBehaviour for TendermintBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); ctx.stream_channel_controller - .broadcast(Event::err( - format!("{}:{}", Self::error_event_name(), ticker), - e, - None, - )) + .broadcast(Event::err(format!("{}:{}", Self::event_name(), ticker), e, None)) .await; continue; diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index c0d89c78f4..f57cbf5a09 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -5,7 +5,7 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::EventBehaviour, ErrorEventName, Event, EventName, Filter}; +use mm2_event_stream::{Event, EventBehaviour, EventName, Filter}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -57,8 +57,6 @@ impl UtxoBalanceEventStreamer { impl EventBehaviour for UtxoBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } - fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; @@ -133,7 +131,7 @@ impl EventBehaviour for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", Self::error_event_name(), coin.ticker()), + format!("{}:{}", Self::event_name(), coin.ticker()), json!({ "error": e }), Some(filter.clone()), )) @@ -152,7 +150,7 @@ impl EventBehaviour for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", Self::error_event_name(), coin.ticker()), + format!("{}:{}", Self::event_name(), coin.ticker()), json!({ "error": e }), Some(filter.clone()), )) @@ -209,7 +207,7 @@ impl EventBehaviour for UtxoBalanceEventStreamer { // FIXME: Note that such an event isn't SSE-ed to any client since no body is listening to it. ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", Self::error_event_name(), ticker), + format!("{}:{}", Self::event_name(), ticker), e, Some(filter.clone()), )) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 97cd28a42a..92e47a09ba 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -14,7 +14,7 @@ use common::log::{debug, error, info, warn}; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::EventBehaviour; #[cfg(test)] use mocktopus::macros::*; use rand::Rng; use script::Builder; diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index cfba2206ce..d1b7502fb4 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -60,7 +60,7 @@ use keys::hash::H256; use keys::{KeyPair, Message, Public}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::EventBehaviour; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; use primitives::bytes::Bytes; diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 256c325714..5cdac57fcd 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -11,8 +11,8 @@ use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::behaviour::EventBehaviour; -use mm2_event_stream::{ErrorEventName, Event, EventName}; +use mm2_event_stream::EventBehaviour; +use mm2_event_stream::{Event, EventName}; use serde_json::Value as Json; use std::sync::Arc; @@ -41,8 +41,6 @@ impl ZCoinBalanceEventStreamer { impl EventBehaviour for ZCoinBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } - fn error_event_name() -> ErrorEventName { ErrorEventName::CoinBalanceError } - async fn handle(self, tx: oneshot::Sender>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let coin = self.coin; @@ -84,7 +82,7 @@ impl EventBehaviour for ZCoinBalanceEventStreamer { }); ctx.stream_channel_controller - .broadcast(Event::err(Self::event_name().to_string(), payload, None)) + .broadcast(Event::new(Self::event_name().to_string(), payload, None)) .await; }, Err(err) => { @@ -93,7 +91,7 @@ impl EventBehaviour for ZCoinBalanceEventStreamer { let e = serde_json::to_value(err).expect("Serialization should't fail."); return ctx .stream_channel_controller - .broadcast(Event::new(format!("{}:{}", Self::error_event_name(), ticker), e, None)) + .broadcast(Event::err(format!("{}:{}", Self::event_name(), ticker), e, None)) .await; }, }; diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 0997322246..06cbfa275e 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -21,7 +21,7 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::EventBehaviour; use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 6742f643cb..5214d1d705 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{controller::Controller, Event, EventStreamConfiguration}; +use mm2_event_stream::{Controller, Event, EventStreamConfiguration}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; @@ -82,6 +82,7 @@ pub struct MmCtx { pub(crate) data_asker: DataAsker, /// Configuration of event streaming used for SSE. pub event_stream_configuration: EventStreamConfiguration, + //pub event_stream_manager: /// True if the MarketMaker instance needs to stop. pub stop: Constructible, /// Unique context identifier, allowing us to more easily pass the context through the FFI boundaries. diff --git a/mm2src/mm2_event_stream/src/behaviour.rs b/mm2src/mm2_event_stream/src/behaviour.rs index bdbc75370d..bfa7d73548 100644 --- a/mm2src/mm2_event_stream/src/behaviour.rs +++ b/mm2src/mm2_event_stream/src/behaviour.rs @@ -1,4 +1,4 @@ -use crate::{ErrorEventName, EventName}; +use crate::EventName; use async_trait::async_trait; use futures::channel::oneshot; @@ -11,10 +11,6 @@ pub trait EventBehaviour { /// No other event should have the same identifier. fn event_id(&self) -> String { unimplemented!() } - /// Returns the name of the error event as an ErrorEventName enum variant. - /// By default, it returns `ErrorEventName::GenericError,` which shows as "ERROR" in the event stream. - fn error_event_name() -> ErrorEventName { ErrorEventName::GenericError } - /// Event handler that is responsible for broadcasting event data to the streaming channels. /// /// `tx` is a oneshot sender that is used to send the initialization status of the event. diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs new file mode 100644 index 0000000000..35cfd40ee6 --- /dev/null +++ b/mm2src/mm2_event_stream/src/configuration.rs @@ -0,0 +1,50 @@ +use std::collections::HashMap; + +use crate::EventName; + +use serde::Deserialize; +use serde_json::Value as Json; + +#[cfg(target_arch = "wasm32")] use std::path::PathBuf; + +#[cfg(target_arch = "wasm32")] +const DEFAULT_WORKER_PATH: &str = "event_streaming_worker.js"; + +/// Configuration for event streaming +#[derive(Deserialize)] +pub struct EventStreamConfiguration { + /// The value to set for the `Access-Control-Allow-Origin` header. + #[serde(default)] + pub access_control_allow_origin: String, + #[serde(default)] + active_events: HashMap, + /// The path to the worker script for event streaming. + #[cfg(target_arch = "wasm32")] + #[serde(default = "default_worker_path")] + pub worker_path: PathBuf, +} + +#[cfg(target_arch = "wasm32")] +#[inline] +fn default_worker_path() -> PathBuf { PathBuf::from(DEFAULT_WORKER_PATH) } + +impl Default for EventStreamConfiguration { + fn default() -> Self { + Self { + access_control_allow_origin: String::from("*"), + active_events: Default::default(), + #[cfg(target_arch = "wasm32")] + worker_path: default_worker_path(), + } + } +} + +impl EventStreamConfiguration { + /// Retrieves the configuration for a specific event by its name. + #[inline] + pub fn get_event(&self, event_name: &EventName) -> Option { self.active_events.get(event_name).cloned() } + + /// Gets the total number of active events in the configuration. + #[inline] + pub fn total_active_events(&self) -> usize { self.active_events.len() } +} diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs new file mode 100644 index 0000000000..76fd106cdb --- /dev/null +++ b/mm2src/mm2_event_stream/src/event.rs @@ -0,0 +1,94 @@ +use std::collections::HashSet; +use std::fmt; +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::Value as Json; + +/// Multi-purpose/generic event type that can easily be used over the event streaming +pub struct Event { + /// The type of the event (balance, network, swap, etc...). + event_type: String, + /// The message to be sent to the client. + message: Json, + /// The filter object to be used to determine whether the event should be sent or not. + /// It could also alter the event content + /// + /// The filter is wrapped in an `Arc` since the event producer should use it as a singleton + /// by using the same filter over and over again with multiple events. + filter: Option>, +} + +impl Event { + /// Creates a new `Event` instance with the specified event type and message. + #[inline] + pub fn new(event_type: String, message: Json, filter: Option>) -> Self { + Self { + event_type, + message, + filter, + } + } + + /// Create a new error `Event` instance with the specified error event type and message. + #[inline] + pub fn err(event_type: String, message: Json, filter: Option>) -> Self { + Self { + event_type: format!("ERROR_{event_type}"), + message, + filter, + } + } + + /// Returns the event type and message to be sent or `None` if the event should not be sent. + /// + /// Uses the `requested_events` to determine whether the event should be sent or not. + /// If `requested_events` is empty, this doesn't mean the event won't be sent, this is + /// decided by the event's filtering mechanism. + /// + /// `requested_events` could also be used to alter the event content (e.g. remove certain fields) + pub fn get_data(&self, requested_events: &HashSet) -> Option<(String, Json)> { + self.filter.as_ref().map_or_else( + // If no filter is set, send the event as is. + || Some((self.event_type.clone(), self.message.clone())), + |filter| { + filter + .filter(&self.message, requested_events) + .map(|message| (self.event_type.clone(), message)) + }, + ) + } +} + +/// A trait that defines the filtering mechanism for events. +/// +/// Each event has a filter that determines whether the event should be send out +/// to the client or not based on the client's requested events. +pub trait Filter: Send + Sync { + /// Filters the event based on the requested events. + /// + /// Returns the (maybe altered) message to be sent or `None` if the event should not be sent. + /// `requested_events` is a set of the events that the client asked to subscribe to (e.g. `BALANCE:BTC`) + fn filter(&self, message: &Json, requested_events: &HashSet) -> Option; +} + +/// Event types streamed to clients through channels like Server-Sent Events (SSE). +#[derive(Deserialize, Eq, Hash, PartialEq)] +pub enum EventName { + /// Indicates a change in the balance of a coin. + BALANCE, + /// Event triggered at regular intervals to indicate that the system is operational. + HEARTBEAT, + /// Returns p2p network information at a regular interval. + NETWORK, +} + +impl fmt::Display for EventName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BALANCE => write!(f, "COIN_BALANCE"), + Self::HEARTBEAT => write!(f, "HEARTBEAT"), + Self::NETWORK => write!(f, "NETWORK"), + } + } +} diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index dd7babdc0c..f1bcbf515b 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,158 +1,11 @@ -use std::collections::{HashMap, HashSet}; -use std::fmt; -use std::sync::Arc; - -use serde::Deserialize; -use serde_json::Value as Json; - -#[cfg(target_arch = "wasm32")] use std::path::PathBuf; - -#[cfg(target_arch = "wasm32")] -const DEFAULT_WORKER_PATH: &str = "event_streaming_worker.js"; - -/// Multi-purpose/generic event type that can easily be used over the event streaming -pub struct Event { - /// The type of the event (balance, network, swap, etc...). - event_type: String, - /// The message to be sent to the client. - message: Json, - /// The filter object to be used to determine whether the event should be sent or not. - /// It could also alter the event content - /// - /// The filter is wrapped in an `Arc` since the event producer should use it as a singleton - /// by using the same filter over and over again with multiple events. - filter: Option>, -} - -impl Event { - /// Creates a new `Event` instance with the specified event type and message. - #[inline] - pub fn new(event_type: String, message: Json, filter: Option>) -> Self { - Self { - event_type, - message, - filter, - } - } - - /// Create a new error `Event` instance with the specified error event type and message. - #[inline] - pub fn err(event_type: String, message: Json, filter: Option>) -> Self { - Self { - event_type: format!("ERROR_{event_type}"), - message, - filter, - } - } - - /// Returns the event type and message to be sent or `None` if the event should not be sent. - /// - /// Uses the `requested_events` to determine whether the event should be sent or not. - /// If `requested_events` is empty, this doesn't mean the event won't be sent, this is - /// decided by the event's filtering mechanism. - /// - /// `requested_events` could also be used to alter the event content (e.g. remove certain fields) - pub fn get_data(&self, requested_events: &HashSet) -> Option<(String, Json)> { - self.filter.as_ref().map_or_else( - // If no filter is set, send the event as is. - || Some((self.event_type.clone(), self.message.clone())), - |filter| { - filter - .filter(&self.message, requested_events) - .map(|message| (self.event_type.clone(), message)) - }, - ) - } -} - -/// A trait that defines the filtering mechanism for events. -/// -/// Each event has a filter that determines whether the event should be send out -/// to the client or not based on the client's requested events. -pub trait Filter: Send + Sync { - /// Filters the event based on the requested events. - /// - /// Returns the (maybe altered) message to be sent or `None` if the event should not be sent. - /// `requested_events` is a set of the events that the client asked to subscribe to (e.g. `BALANCE:BTC`) - fn filter(&self, message: &Json, requested_events: &HashSet) -> Option; -} - -/// Event types streamed to clients through channels like Server-Sent Events (SSE). -#[derive(Deserialize, Eq, Hash, PartialEq)] -pub enum EventName { - /// Indicates a change in the balance of a coin. - BALANCE, - /// Event triggered at regular intervals to indicate that the system is operational. - HEARTBEAT, - /// Returns p2p network information at a regular interval. - NETWORK, -} - -impl fmt::Display for EventName { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::BALANCE => write!(f, "COIN_BALANCE"), - Self::HEARTBEAT => write!(f, "HEARTBEAT"), - Self::NETWORK => write!(f, "NETWORK"), - } - } -} - -/// Error event types used to indicate various kinds of errors to clients through channels like Server-Sent Events (SSE). -pub enum ErrorEventName { - /// A generic error that doesn't fit any other specific categories. - GenericError, - /// Signifies an error related to fetching or calculating the balance of a coin. - CoinBalanceError, -} - -impl fmt::Display for ErrorEventName { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::GenericError => write!(f, "ERROR"), - Self::CoinBalanceError => write!(f, "COIN_BALANCE_ERROR"), - } - } -} - -/// Configuration for event streaming -#[derive(Deserialize)] -pub struct EventStreamConfiguration { - /// The value to set for the `Access-Control-Allow-Origin` header. - #[serde(default)] - pub access_control_allow_origin: String, - #[serde(default)] - active_events: HashMap, - /// The path to the worker script for event streaming. - #[cfg(target_arch = "wasm32")] - #[serde(default = "default_worker_path")] - pub worker_path: PathBuf, -} - -#[cfg(target_arch = "wasm32")] -#[inline] -fn default_worker_path() -> PathBuf { PathBuf::from(DEFAULT_WORKER_PATH) } - -impl Default for EventStreamConfiguration { - fn default() -> Self { - Self { - access_control_allow_origin: String::from("*"), - active_events: Default::default(), - #[cfg(target_arch = "wasm32")] - worker_path: default_worker_path(), - } - } -} - -impl EventStreamConfiguration { - /// Retrieves the configuration for a specific event by its name. - #[inline] - pub fn get_event(&self, event_name: &EventName) -> Option { self.active_events.get(event_name).cloned() } - - /// Gets the total number of active events in the configuration. - #[inline] - pub fn total_active_events(&self) -> usize { self.active_events.len() } -} - pub mod behaviour; +pub mod configuration; pub mod controller; +pub mod event; +pub mod manager; + +// Re-export important types. +pub use behaviour::EventBehaviour; +pub use configuration::EventStreamConfiguration; +pub use controller::Controller; +pub use event::{Event, EventName, Filter}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs new file mode 100644 index 0000000000..46d17f8176 --- /dev/null +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -0,0 +1,7 @@ +// use std::collections::HashMap; + +// use crate::EventBehaviour; + +// pub struct StreamingManager { +// streamers: HashMap>, +// } diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 7fbde103a8..864b0eba5f 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -3,7 +3,7 @@ use common::{executor::{SpawnFuture, Timer}, log::info}; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{behaviour::EventBehaviour, Event, EventName}; +use mm2_event_stream::{Event, EventBehaviour, EventName}; use serde::Deserialize; use serde_json::Value as Json; diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 84c78e6029..1ccac45b3f 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -28,7 +28,7 @@ use enum_derives::EnumFromTrait; use mm2_core::mm_ctx::{MmArc, MmCtx}; use mm2_err_handle::common_errors::InternalError; use mm2_err_handle::prelude::*; -use mm2_event_stream::behaviour::EventBehaviour; +use mm2_event_stream::EventBehaviour; use mm2_libp2p::behaviours::atomicdex::{GossipsubConfig, DEPRECATED_NETID_LIST}; use mm2_libp2p::{spawn_gossipsub, AdexBehaviourError, NodeType, RelayAddress, RelayAddressError, SeedNodeInfo, SwarmRuntime, WssCerts}; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 0acff08d6e..1e8ae81e1d 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -4,7 +4,7 @@ use common::{executor::{SpawnFuture, Timer}, log::info}; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -pub use mm2_event_stream::behaviour::EventBehaviour; +pub use mm2_event_stream::EventBehaviour; use mm2_event_stream::{Event, EventName}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; From 209df5fb1b7379052b379aa74bb6d4439841629d Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 12 Jul 2024 19:26:46 +0200 Subject: [PATCH 016/128] rename event behavior to event streamer --- mm2src/coins/eth.rs | 2 +- mm2src/coins/eth/eth_balance_events.rs | 4 ++-- mm2src/coins/tendermint/tendermint_balance_events.rs | 4 ++-- mm2src/coins/utxo/utxo_balance_events.rs | 4 ++-- mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs | 2 +- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_balance_streaming.rs | 4 ++-- .../coins_activation/src/tendermint_with_assets_activation.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 4 ++-- mm2src/mm2_event_stream/src/{behaviour.rs => streamer.rs} | 2 +- mm2src/mm2_main/src/heartbeat_event.rs | 4 ++-- mm2src/mm2_main/src/lp_native_dex.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 4 ++-- 13 files changed, 20 insertions(+), 20 deletions(-) rename mm2src/mm2_event_stream/src/{behaviour.rs => streamer.rs} (96%) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 61c2c5354a..35d8282023 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -79,7 +79,7 @@ use http::Uri; use instant::Instant; use keys::Public as HtlcPubKey; use mm2_core::mm_ctx::{MmArc, MmWeak}; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; use mm2_net::transport::{GuiAuthValidation, GuiAuthValidationGenerator}; use mm2_number::bigdecimal_custom::CheckedDivision; use mm2_number::{BigDecimal, BigUint, MmNumber}; diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 15285e604f..0744f34e85 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -6,7 +6,7 @@ use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Event, EventBehaviour, EventName}; +use mm2_event_stream::{Event, EventStreamer, EventName}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -147,7 +147,7 @@ async fn fetch_balance( } #[async_trait] -impl EventBehaviour for EthBalanceEventStreamer { +impl EventStreamer for EthBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } async fn handle(self, tx: oneshot::Sender>) { diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 0f6f9fe7dd..c104dfe356 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -6,7 +6,7 @@ use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventBehaviour, EventName}; +use mm2_event_stream::{Event, EventStreamer, EventName}; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; @@ -34,7 +34,7 @@ impl TendermintBalanceEventStreamer { } #[async_trait] -impl EventBehaviour for TendermintBalanceEventStreamer { +impl EventStreamer for TendermintBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } async fn handle(self, tx: oneshot::Sender>) { diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index f57cbf5a09..5978b4c092 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -5,7 +5,7 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventBehaviour, EventName, Filter}; +use mm2_event_stream::{Event, EventStreamer, EventName, Filter}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -54,7 +54,7 @@ impl UtxoBalanceEventStreamer { } #[async_trait] -impl EventBehaviour for UtxoBalanceEventStreamer { +impl EventStreamer for UtxoBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } async fn handle(self, tx: oneshot::Sender>) { diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 92e47a09ba..b6d65f86e6 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -14,7 +14,7 @@ use common::log::{debug, error, info, warn}; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; #[cfg(test)] use mocktopus::macros::*; use rand::Rng; use script::Builder; diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index d1b7502fb4..ae16598b39 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -60,7 +60,7 @@ use keys::hash::H256; use keys::{KeyPair, Message, Public}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; use primitives::bytes::Bytes; diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 5cdac57fcd..64168bbfc3 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -11,7 +11,7 @@ use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; use mm2_event_stream::{Event, EventName}; use serde_json::Value as Json; use std::sync::Arc; @@ -38,7 +38,7 @@ impl ZCoinBalanceEventStreamer { } #[async_trait] -impl EventBehaviour for ZCoinBalanceEventStreamer { +impl EventStreamer for ZCoinBalanceEventStreamer { fn event_name() -> EventName { EventName::BALANCE } async fn handle(self, tx: oneshot::Sender>) { diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 06cbfa275e..57a7f45dd1 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -21,7 +21,7 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index f1bcbf515b..0bbfb46fd5 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,11 +1,11 @@ -pub mod behaviour; +pub mod streamer; pub mod configuration; pub mod controller; pub mod event; pub mod manager; // Re-export important types. -pub use behaviour::EventBehaviour; +pub use streamer::EventStreamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, EventName, Filter}; diff --git a/mm2src/mm2_event_stream/src/behaviour.rs b/mm2src/mm2_event_stream/src/streamer.rs similarity index 96% rename from mm2src/mm2_event_stream/src/behaviour.rs rename to mm2src/mm2_event_stream/src/streamer.rs index bfa7d73548..48fd854c52 100644 --- a/mm2src/mm2_event_stream/src/behaviour.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use futures::channel::oneshot; #[async_trait] -pub trait EventBehaviour { +pub trait EventStreamer { /// Returns the name of the event as an EventName enum variant. fn event_name() -> EventName; diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 864b0eba5f..0937beba26 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -3,7 +3,7 @@ use common::{executor::{SpawnFuture, Timer}, log::info}; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventBehaviour, EventName}; +use mm2_event_stream::{Event, EventStreamer, EventName}; use serde::Deserialize; use serde_json::Value as Json; @@ -32,7 +32,7 @@ impl HeartbeatEvent { } #[async_trait] -impl EventBehaviour for HeartbeatEvent { +impl EventStreamer for HeartbeatEvent { fn event_name() -> EventName { EventName::HEARTBEAT } async fn handle(self, tx: oneshot::Sender>) { diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 1ccac45b3f..2d27cede21 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -28,7 +28,7 @@ use enum_derives::EnumFromTrait; use mm2_core::mm_ctx::{MmArc, MmCtx}; use mm2_err_handle::common_errors::InternalError; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventBehaviour; +use mm2_event_stream::EventStreamer; use mm2_libp2p::behaviours::atomicdex::{GossipsubConfig, DEPRECATED_NETID_LIST}; use mm2_libp2p::{spawn_gossipsub, AdexBehaviourError, NodeType, RelayAddress, RelayAddressError, SeedNodeInfo, SwarmRuntime, WssCerts}; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 1e8ae81e1d..e72587c616 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -4,7 +4,7 @@ use common::{executor::{SpawnFuture, Timer}, log::info}; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -pub use mm2_event_stream::EventBehaviour; +pub use mm2_event_stream::EventStreamer; use mm2_event_stream::{Event, EventName}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; @@ -38,7 +38,7 @@ impl NetworkEvent { } #[async_trait] -impl EventBehaviour for NetworkEvent { +impl EventStreamer for NetworkEvent { fn event_name() -> EventName { EventName::NETWORK } async fn handle(self, tx: oneshot::Sender>) { From 4d15a66fcf5bb78ac0ddc0d88bc0b663a5788d69 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 15 Jul 2024 10:35:21 +0200 Subject: [PATCH 017/128] new event streaming arch --- mm2src/mm2_event_stream/src/configuration.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 4 +- mm2src/mm2_event_stream/src/manager.rs | 66 +++++++++++- mm2src/mm2_event_stream/src/streamer.rs | 103 +++++++++++++++++-- 4 files changed, 157 insertions(+), 18 deletions(-) diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs index 35cfd40ee6..786e169db9 100644 --- a/mm2src/mm2_event_stream/src/configuration.rs +++ b/mm2src/mm2_event_stream/src/configuration.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use crate::EventName; +use super::EventName; use serde::Deserialize; use serde_json::Value as Json; diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 0bbfb46fd5..cdbb973b43 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,11 +1,11 @@ -pub mod streamer; pub mod configuration; pub mod controller; pub mod event; pub mod manager; +pub mod streamer; // Re-export important types. -pub use streamer::EventStreamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, EventName, Filter}; +pub use streamer::{EventStreamer, NoDataIn}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 46d17f8176..3e23755bf5 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,7 +1,63 @@ -// use std::collections::HashMap; +use std::any::Any; +use std::collections::HashMap; -// use crate::EventBehaviour; +use super::EventStreamer; +use common::executor::abortable_queue::WeakSpawner; -// pub struct StreamingManager { -// streamers: HashMap>, -// } +use futures::channel::mpsc::UnboundedSender; +use futures::channel::oneshot; + +/// The errors that could originate from the streaming manager. +pub enum StreamingSendError { + /// There is no streamer with the given ID. + StreamerNotFound, + /// Couldn't send the data to the streamer. + SendError, + /// The streamer doesn't accept an input. + NoDataIn, +} + +pub struct StreamingManager { + /// A map from streamer IDs to their communication channels (if present) and shutdown handles. + streamers: HashMap>>, oneshot::Sender<()>)>, +} + +impl StreamingManager { + /// Spawns and adds a new streamer `streamer` to the manager. + pub async fn add(&mut self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), String> { + let streamer_id = streamer.streamer_id(); + // Spawn the streamer. + streamer.spawn(spawner).await.map(|(shutdown_signal, data_in_channel)| { + // And store its handles if spawned successfully. + self.streamers + .insert(streamer_id.clone(), (data_in_channel, shutdown_signal)) + // Two different streamers shouldn't conflict if IDs are unique, so this is a bug. + .map(|_| { + common::log::error!( + "A streamer with the same id ({}) existed and now has been removed.", + streamer_id + ) + }); + }) + } + + /// Sends data to a streamer of a known ID. + pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingSendError> { + let (data_in, _) = self + .streamers + .get(streamer_id) + .ok_or(StreamingSendError::StreamerNotFound)?; + let data_in = data_in.as_ref().ok_or(StreamingSendError::NoDataIn)?; + data_in + .unbounded_send(Box::new(data)) + .map_err(|_| StreamingSendError::SendError) + } + + /// Shuts down a streamer of a known ID. + pub fn shut(&mut self, streamer_id: &str) -> Result<(), StreamingSendError> { + self.streamers + .remove(streamer_id) + .ok_or(StreamingSendError::StreamerNotFound)?; + Ok(()) + } +} diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 48fd854c52..e28d97c5c0 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,21 +1,104 @@ -use crate::EventName; +use std::any::{self, Any}; + +use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable, SpawnFuture}; +use common::log::{error, info}; + use async_trait::async_trait; -use futures::channel::oneshot; +use futures::channel::{mpsc, oneshot}; +use futures::StreamExt; + +/// A marker to indicate that the event streamer doesn't take any input data. +pub struct NoDataIn; #[async_trait] -pub trait EventStreamer { - /// Returns the name of the event as an EventName enum variant. - fn event_name() -> EventName; +pub trait EventStreamer +where + Self: Sized + Send + 'static, +{ + type DataInType: Send; /// Returns a human readable unique identifier for the event streamer. - /// No other event should have the same identifier. - fn event_id(&self) -> String { unimplemented!() } + /// No other event streamer should have the same identifier. + fn streamer_id(&self) -> String { unimplemented!() } /// Event handler that is responsible for broadcasting event data to the streaming channels. /// - /// `tx` is a oneshot sender that is used to send the initialization status of the event. - async fn handle(self, tx: oneshot::Sender>); + /// `ready_tx` is a oneshot sender that is used to send the initialization status of the event. + /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. + async fn handle( + self, + ready_tx: oneshot::Sender>, + data_rx: mpsc::UnboundedReceiver, + ); /// Spawns the `Self::handle` in a separate thread. - async fn spawn(self) -> Result<(), String>; + /// + /// Returns a `oneshot::Sender` to shutdown the handler and an optional `mpsc::UnboundedSender` + /// to send data to the handler. + /// + /// This method should not be overridden. + async fn spawn( + self, + // FIXME: Might be better to let the implementors store the spawner themselves and + // we can use `self.spawner()` here to get it. + // Also for AbortSettings, we can make this customizable with a default impl. + spawner: WeakSpawner, + ) -> Result<(oneshot::Sender<()>, Option>>), String> { + let streamer_id = self.streamer_id(); + info!("Spawning event streamer: {streamer_id}"); + + // A oneshot channel to receive the initialization status of the handler through. + let (tx_ready, ready_rx) = oneshot::channel(); + // A oneshot channel to shutdown the handler. + let (tx_shutdown, rx_shutdown) = oneshot::channel::<()>(); + // An unbounded channel to send data to the handler. + let (data_sender, data_receiver) = mpsc::unbounded(); + + // FIXME: select! between shutdown or handle here. + let handler_with_shutdown = async move { + drop(rx_shutdown); + self.handle(tx_ready, data_receiver).await; + }; + let settings = AbortSettings::info_on_abort(format!("{streamer_id} streamer has stopped.")); + spawner.spawn_with_settings(handler_with_shutdown, settings); + + ready_rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + })?; + + // If the handler takes no input data, return `None` for the data sender. + if any::TypeId::of::() == any::TypeId::of::() { + Ok((tx_shutdown, None)) + } else { + let (any_data_sender, mut rx) = mpsc::unbounded::>(); + + // To store the data senders in the `StreamingManager`, they have to be casted to the same type + // (`UnboundedSender>`). But for convenience, we want the implementors of the trait + // to think that the sending channel is of type `UnboundedSender`. + // This spawned task will listen to the incoming `Box` data and convert it to `Self::DataInType`. + // + // One other solution is to let `Self::handle` take a `UnboundedReceiver>` and cast + // it internally, but this means this (confusing) boiler plate will need to be done for each + // implementation of this trait. + // + // FIXME: Check if something like `rx_to_stream` (rpc_clients.rs) could work without an extra thread. + spawner.spawn(async move { + while let Some(any_input_data) = rx.next().await { + if let Ok(input_data) = any_input_data.downcast() { + if let Err(_) = data_sender.unbounded_send(*input_data) { + // The handler dropped the data receiver. + return; + } + } + else { + error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); + } + } + }); + + Ok((tx_shutdown, Some(any_data_sender))) + } + } } From 009d206f087899da006bbcc99c20586e0a96ace4 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 15 Jul 2024 11:44:53 +0200 Subject: [PATCH 018/128] event streamer optimizations and fixmes remove the intermediate bridge task and replace it with `filter_map` call. also properly implement the shutdown handle --- mm2src/mm2_event_stream/src/streamer.rs | 71 ++++++++++++------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index e28d97c5c0..f14aeae115 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,15 +1,20 @@ use std::any::{self, Any}; -use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable, SpawnFuture}; +use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; use async_trait::async_trait; use futures::channel::{mpsc, oneshot}; -use futures::StreamExt; +use futures::{select, FutureExt, Stream, StreamExt}; /// A marker to indicate that the event streamer doesn't take any input data. pub struct NoDataIn; +/// Helper function casting mpsc::Receiver as Stream. +fn rx_to_stream(rx: mpsc::UnboundedReceiver) -> impl Stream { + rx //.map_err(|_| panic!("errors not possible on rx")) +} + #[async_trait] pub trait EventStreamer where @@ -25,11 +30,7 @@ where /// /// `ready_tx` is a oneshot sender that is used to send the initialization status of the event. /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. - async fn handle( - self, - ready_tx: oneshot::Sender>, - data_rx: mpsc::UnboundedReceiver, - ); + async fn handle(self, ready_tx: oneshot::Sender>, data_rx: impl Stream); /// Spawns the `Self::handle` in a separate thread. /// @@ -52,12 +53,32 @@ where // A oneshot channel to shutdown the handler. let (tx_shutdown, rx_shutdown) = oneshot::channel::<()>(); // An unbounded channel to send data to the handler. - let (data_sender, data_receiver) = mpsc::unbounded(); + let (any_data_sender, any_data_receiver) = mpsc::unbounded::>(); + // A middleware to cast the data of type `Box` to the actual input datatype of this streamer. + let data_receiver = rx_to_stream(any_data_receiver).filter_map({ + //let err_msg = format!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); + move |any_input_data| async move { + if let Ok(input_data) = any_input_data.downcast::() { + Some(*input_data) + } else { + // FIXME: Can't use `streamer_id` here. + //error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); + //error!("{err_msg}"); + None + } + } + }); - // FIXME: select! between shutdown or handle here. - let handler_with_shutdown = async move { - drop(rx_shutdown); - self.handle(tx_ready, data_receiver).await; + let handler_with_shutdown = { + let streamer_id = streamer_id.clone(); + async move { + select! { + _ = rx_shutdown.fuse() => { + info!("Manually shutting down event streamer: {streamer_id}.") + } + _ = self.handle(tx_ready, data_receiver).fuse() => {} + } + } }; let settings = AbortSettings::info_on_abort(format!("{streamer_id} streamer has stopped.")); spawner.spawn_with_settings(handler_with_shutdown, settings); @@ -72,32 +93,6 @@ where if any::TypeId::of::() == any::TypeId::of::() { Ok((tx_shutdown, None)) } else { - let (any_data_sender, mut rx) = mpsc::unbounded::>(); - - // To store the data senders in the `StreamingManager`, they have to be casted to the same type - // (`UnboundedSender>`). But for convenience, we want the implementors of the trait - // to think that the sending channel is of type `UnboundedSender`. - // This spawned task will listen to the incoming `Box` data and convert it to `Self::DataInType`. - // - // One other solution is to let `Self::handle` take a `UnboundedReceiver>` and cast - // it internally, but this means this (confusing) boiler plate will need to be done for each - // implementation of this trait. - // - // FIXME: Check if something like `rx_to_stream` (rpc_clients.rs) could work without an extra thread. - spawner.spawn(async move { - while let Some(any_input_data) = rx.next().await { - if let Ok(input_data) = any_input_data.downcast() { - if let Err(_) = data_sender.unbounded_send(*input_data) { - // The handler dropped the data receiver. - return; - } - } - else { - error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); - } - } - }); - Ok((tx_shutdown, Some(any_data_sender))) } } From 7a6c224f550aa9f8e26ad052d516865e38944d43 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 15 Jul 2024 18:43:58 +0200 Subject: [PATCH 019/128] fix not printing streamer id on downcast error --- mm2src/mm2_event_stream/src/manager.rs | 6 +++--- mm2src/mm2_event_stream/src/streamer.rs | 27 ++++++++++++------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 3e23755bf5..5f7f81210a 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -19,7 +19,7 @@ pub enum StreamingSendError { pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. - streamers: HashMap>>, oneshot::Sender<()>)>, + streamers: HashMap, Option>>)>, } impl StreamingManager { @@ -30,7 +30,7 @@ impl StreamingManager { streamer.spawn(spawner).await.map(|(shutdown_signal, data_in_channel)| { // And store its handles if spawned successfully. self.streamers - .insert(streamer_id.clone(), (data_in_channel, shutdown_signal)) + .insert(streamer_id.clone(), (shutdown_signal, data_in_channel)) // Two different streamers shouldn't conflict if IDs are unique, so this is a bug. .map(|_| { common::log::error!( @@ -43,7 +43,7 @@ impl StreamingManager { /// Sends data to a streamer of a known ID. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingSendError> { - let (data_in, _) = self + let (_, data_in) = self .streamers .get(streamer_id) .ok_or(StreamingSendError::StreamerNotFound)?; diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index f14aeae115..4ae05b4909 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -10,10 +10,8 @@ use futures::{select, FutureExt, Stream, StreamExt}; /// A marker to indicate that the event streamer doesn't take any input data. pub struct NoDataIn; -/// Helper function casting mpsc::Receiver as Stream. -fn rx_to_stream(rx: mpsc::UnboundedReceiver) -> impl Stream { - rx //.map_err(|_| panic!("errors not possible on rx")) -} +/// Helper function to cast mpsc::UnboundedReceiver to impl Stream. +fn rx_to_stream(rx: mpsc::UnboundedReceiver) -> impl Stream { rx } #[async_trait] pub trait EventStreamer @@ -24,7 +22,7 @@ where /// Returns a human readable unique identifier for the event streamer. /// No other event streamer should have the same identifier. - fn streamer_id(&self) -> String { unimplemented!() } + fn streamer_id(&self) -> String; /// Event handler that is responsible for broadcasting event data to the streaming channels. /// @@ -56,15 +54,16 @@ where let (any_data_sender, any_data_receiver) = mpsc::unbounded::>(); // A middleware to cast the data of type `Box` to the actual input datatype of this streamer. let data_receiver = rx_to_stream(any_data_receiver).filter_map({ - //let err_msg = format!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); - move |any_input_data| async move { - if let Ok(input_data) = any_input_data.downcast::() { - Some(*input_data) - } else { - // FIXME: Can't use `streamer_id` here. - //error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); - //error!("{err_msg}"); - None + let streamer_id = streamer_id.clone(); + move |any_input_data| { + let streamer_id = streamer_id.clone(); + async move { + if let Ok(input_data) = any_input_data.downcast() { + Some(*input_data) + } else { + error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); + None + } } } }); From 8fa5950a5fd1616190620aac74a8ccd0fbcd9ee6 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 15 Jul 2024 19:33:56 +0200 Subject: [PATCH 020/128] refuse to add a streamer that is already present also wraps the streamermanager mutable data inside Arc, Option>>)>, + streamers: Arc, Option>>)>>>, } impl StreamingManager { /// Spawns and adds a new streamer `streamer` to the manager. - pub async fn add(&mut self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), String> { - let streamer_id = streamer.streamer_id(); - // Spawn the streamer. - streamer.spawn(spawner).await.map(|(shutdown_signal, data_in_channel)| { - // And store its handles if spawned successfully. - self.streamers - .insert(streamer_id.clone(), (shutdown_signal, data_in_channel)) - // Two different streamers shouldn't conflict if IDs are unique, so this is a bug. - .map(|_| { - common::log::error!( - "A streamer with the same id ({}) existed and now has been removed.", - streamer_id - ) - }); - }) + pub async fn add(&self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), String> { + let streamer_id = streamer.streamer_id().to_string(); + // NOTE: We spawn the streamer *before* checking if it can be added or not because + // we don't know how much time will it take for spawning and we don't want to lock + // the manager for too long. + let channels = streamer.spawn(spawner).await?; + let mut streamers = self.streamers.write().unwrap(); + // If that streamer already exists, refuse to add it. + if streamers.contains_key(&streamer_id) { + return Err(format!( + "A streamer with the same id ({streamer_id}) exists, it must be shutdown before re-using the same id." + )); + } + streamers.insert(streamer_id, channels); + Ok(()) } /// Sends data to a streamer of a known ID. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingSendError> { - let (_, data_in) = self - .streamers - .get(streamer_id) - .ok_or(StreamingSendError::StreamerNotFound)?; + let streamers = self.streamers.read().unwrap(); + let (_, data_in) = streamers.get(streamer_id).ok_or(StreamingSendError::StreamerNotFound)?; let data_in = data_in.as_ref().ok_or(StreamingSendError::NoDataIn)?; data_in .unbounded_send(Box::new(data)) @@ -54,8 +54,10 @@ impl StreamingManager { } /// Shuts down a streamer of a known ID. - pub fn shut(&mut self, streamer_id: &str) -> Result<(), StreamingSendError> { + pub fn shut(&self, streamer_id: &str) -> Result<(), StreamingSendError> { self.streamers + .write() + .unwrap() .remove(streamer_id) .ok_or(StreamingSendError::StreamerNotFound)?; Ok(()) diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 4ae05b4909..5ccda9fa05 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -22,7 +22,7 @@ where /// Returns a human readable unique identifier for the event streamer. /// No other event streamer should have the same identifier. - fn streamer_id(&self) -> String; + fn streamer_id(&self) -> &str; /// Event handler that is responsible for broadcasting event data to the streaming channels. /// @@ -43,7 +43,7 @@ where // Also for AbortSettings, we can make this customizable with a default impl. spawner: WeakSpawner, ) -> Result<(oneshot::Sender<()>, Option>>), String> { - let streamer_id = self.streamer_id(); + let streamer_id = self.streamer_id().to_string(); info!("Spawning event streamer: {streamer_id}"); // A oneshot channel to receive the initialization status of the handler through. From 38e22c338c3835aaa8a20c2be66783d8d1b640a0 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 15 Jul 2024 20:48:16 +0200 Subject: [PATCH 021/128] adapt network and heartbeat streamers to the new interface along with some cleanups and simplifications --- mm2src/mm2_core/src/mm_ctx.rs | 6 ++-- mm2src/mm2_event_stream/src/configuration.rs | 6 ++-- mm2src/mm2_event_stream/src/lib.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 2 +- mm2src/mm2_event_stream/src/streamer.rs | 14 +++++--- mm2src/mm2_main/src/heartbeat_event.rs | 30 ++++------------- mm2src/mm2_main/src/lp_native_dex.rs | 18 ++++++----- mm2src/mm2_net/src/network_event.rs | 34 +++++--------------- 8 files changed, 43 insertions(+), 69 deletions(-) diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 5214d1d705..66f9fcd0d1 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{Controller, Event, EventStreamConfiguration}; +use mm2_event_stream::{manager::StreamingManager, Controller, Event, EventStreamConfiguration}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; @@ -82,7 +82,8 @@ pub struct MmCtx { pub(crate) data_asker: DataAsker, /// Configuration of event streaming used for SSE. pub event_stream_configuration: EventStreamConfiguration, - //pub event_stream_manager: + /// A manager for the event streaming system. To be used to start/stop/communicate with event streamers. + pub event_stream_manager: StreamingManager, /// True if the MarketMaker instance needs to stop. pub stop: Constructible, /// Unique context identifier, allowing us to more easily pass the context through the FFI boundaries. @@ -157,6 +158,7 @@ impl MmCtx { stream_channel_controller: Controller::new(), data_asker: DataAsker::default(), event_stream_configuration: Default::default(), + event_stream_manager: Default::default(), stop: Constructible::default(), ffi_handle: Constructible::default(), ordermatch_ctx: Mutex::new(None), diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs index 786e169db9..b34707ecb9 100644 --- a/mm2src/mm2_event_stream/src/configuration.rs +++ b/mm2src/mm2_event_stream/src/configuration.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; -use super::EventName; - use serde::Deserialize; use serde_json::Value as Json; @@ -17,7 +15,7 @@ pub struct EventStreamConfiguration { #[serde(default)] pub access_control_allow_origin: String, #[serde(default)] - active_events: HashMap, + active_events: HashMap, /// The path to the worker script for event streaming. #[cfg(target_arch = "wasm32")] #[serde(default = "default_worker_path")] @@ -42,7 +40,7 @@ impl Default for EventStreamConfiguration { impl EventStreamConfiguration { /// Retrieves the configuration for a specific event by its name. #[inline] - pub fn get_event(&self, event_name: &EventName) -> Option { self.active_events.get(event_name).cloned() } + pub fn get_event(&self, event_name: &str) -> Option { self.active_events.get(event_name).cloned() } /// Gets the total number of active events in the configuration. #[inline] diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index cdbb973b43..ddec11d74e 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -8,4 +8,4 @@ pub mod streamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, EventName, Filter}; -pub use streamer::{EventStreamer, NoDataIn}; +pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 0a642eabae..94516d17bd 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -18,7 +18,7 @@ pub enum StreamingSendError { NoDataIn, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: Arc, Option>>)>>>, diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 5ccda9fa05..e3db8a62c4 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -10,8 +10,10 @@ use futures::{select, FutureExt, Stream, StreamExt}; /// A marker to indicate that the event streamer doesn't take any input data. pub struct NoDataIn; -/// Helper function to cast mpsc::UnboundedReceiver to impl Stream. -fn rx_to_stream(rx: mpsc::UnboundedReceiver) -> impl Stream { rx } +/// A mixture trait combining `Stream` and `Send` together (to avoid confusing annotation). +pub trait StreamHandlerInput: Stream + Send {} +/// Implement the trait for all types `T` that implement `Stream + Send` for all `D`. +impl StreamHandlerInput for T where T: Stream + Send {} #[async_trait] pub trait EventStreamer @@ -28,7 +30,11 @@ where /// /// `ready_tx` is a oneshot sender that is used to send the initialization status of the event. /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. - async fn handle(self, ready_tx: oneshot::Sender>, data_rx: impl Stream); + async fn handle( + self, + ready_tx: oneshot::Sender>, + data_rx: impl StreamHandlerInput, + ); /// Spawns the `Self::handle` in a separate thread. /// @@ -53,7 +59,7 @@ where // An unbounded channel to send data to the handler. let (any_data_sender, any_data_receiver) = mpsc::unbounded::>(); // A middleware to cast the data of type `Box` to the actual input datatype of this streamer. - let data_receiver = rx_to_stream(any_data_receiver).filter_map({ + let data_receiver = any_data_receiver.filter_map({ let streamer_id = streamer_id.clone(); move |any_input_data| { let streamer_id = streamer_id.clone(); diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 0937beba26..6e0e84f2e1 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -1,9 +1,8 @@ use async_trait::async_trait; -use common::{executor::{SpawnFuture, Timer}, - log::info}; +use common::executor::Timer; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, EventName}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; use serde::Deserialize; use serde_json::Value as Json; @@ -33,35 +32,20 @@ impl HeartbeatEvent { #[async_trait] impl EventStreamer for HeartbeatEvent { - fn event_name() -> EventName { EventName::HEARTBEAT } + type DataInType = NoDataIn; - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> &str { "HEARTBEAT" } + + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { tx.send(Ok(())).unwrap(); loop { self.ctx .stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), json!({}), None)) + .broadcast(Event::new(self.streamer_id().to_string(), json!({}), None)) .await; Timer::sleep(self.config.stream_interval_seconds).await; } } - - async fn spawn(self) -> Result<(), String> { - info!( - "{} event is activated with config: {:?}", - Self::event_name(), - self.config - ); - - let (tx, rx) = oneshot::channel(); - self.ctx.spawner().spawn(self.handle(tx)); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler dropped before sending an initialization status: {e}", - )) - }) - } } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 2d27cede21..6dfdd23d6c 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -426,18 +426,20 @@ fn migrate_db(ctx: &MmArc) -> MmInitResult<()> { fn migration_1(_ctx: &MmArc) {} async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { - if let Some(config) = ctx.event_stream_configuration.get_event(&NetworkEvent::event_name()) { - NetworkEvent::try_new(config, ctx.clone()) - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))? - .spawn() + if let Some(config) = ctx.event_stream_configuration.get_event("NETWORK") { + let network_steamer = NetworkEvent::try_new(config, ctx.clone()) + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))?; + ctx.event_stream_manager + .add(network_steamer, ctx.spawner().weak()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e}")))?; } - if let Some(config) = ctx.event_stream_configuration.get_event(&HeartbeatEvent::event_name()) { - HeartbeatEvent::try_new(config, ctx.clone()) - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")))? - .spawn() + if let Some(config) = ctx.event_stream_configuration.get_event("HEARTBEAT") { + let heartbeat_streamer = HeartbeatEvent::try_new(config, ctx.clone()) + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")))?; + ctx.event_stream_manager + .add(heartbeat_streamer, ctx.spawner().weak()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e}")))?; } diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index e72587c616..d93eb6bf8b 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -1,11 +1,9 @@ use crate::p2p::P2PContext; use async_trait::async_trait; -use common::{executor::{SpawnFuture, Timer}, - log::info}; +use common::executor::Timer; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -pub use mm2_event_stream::EventStreamer; -use mm2_event_stream::{Event, EventName}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; use serde_json::{json, Value as Json}; @@ -39,13 +37,15 @@ impl NetworkEvent { #[async_trait] impl EventStreamer for NetworkEvent { - fn event_name() -> EventName { EventName::NETWORK } + type DataInType = NoDataIn; - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> &str { "NETWORK" } + + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { let p2p_ctx = P2PContext::fetch_from_mm_arc(&self.ctx); let mut previously_sent = json!({}); - tx.send(Ok(())).unwrap(); + ready_tx.send(Ok(())).unwrap(); loop { let p2p_cmd_tx = p2p_ctx.cmd_tx.lock().clone(); @@ -67,7 +67,7 @@ impl EventStreamer for NetworkEvent { if previously_sent != event_data || self.config.always_send { self.ctx .stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), event_data.clone(), None)) + .broadcast(Event::new(self.streamer_id().to_string(), event_data.clone(), None)) .await; previously_sent = event_data; @@ -76,22 +76,4 @@ impl EventStreamer for NetworkEvent { Timer::sleep(self.config.stream_interval_seconds).await; } } - - async fn spawn(self) -> Result<(), String> { - info!( - "{} event is activated with config: {:?}", - Self::event_name(), - self.config - ); - - let (tx, rx) = oneshot::channel(); - self.ctx.spawner().spawn(self.handle(tx)); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler dropped before sending an initialization status: {}", - e - )) - }) - } } From 945463f3202bbbe43a51290477a0da4177d28dbe Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 12:25:44 +0200 Subject: [PATCH 022/128] adapt the rest of the streamers to the new arch --- mm2src/coins/eth.rs | 17 ++--- mm2src/coins/eth/eth_balance_events.rs | 53 +++----------- .../tendermint/tendermint_balance_events.rs | 34 ++------- mm2src/coins/tendermint/tendermint_coin.rs | 2 +- mm2src/coins/utxo/utxo_balance_events.rs | 73 ++++++------------- .../utxo/utxo_builder/utxo_arc_builder.rs | 29 +++----- mm2src/coins/z_coin.rs | 18 ++--- mm2src/coins/z_coin/z_balance_streaming.rs | 40 ++-------- .../src/tendermint_with_assets_activation.rs | 44 ++++++----- mm2src/mm2_event_stream/src/manager.rs | 4 +- mm2src/mm2_event_stream/src/streamer.rs | 4 +- mm2src/mm2_main/src/heartbeat_event.rs | 6 +- mm2src/mm2_main/src/lp_native_dex.rs | 1 + mm2src/mm2_net/src/network_event.rs | 6 +- 14 files changed, 107 insertions(+), 224 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 35d8282023..0b1bf6687b 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5283,17 +5283,12 @@ impl EthCoin { } async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - if let Some(config) = ctx - .event_stream_configuration - .get_event(&EthBalanceEventStreamer::event_name()) - { - EthBalanceEventStreamer::try_new(config, self.clone()) - .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))? - .spawn() - .await - .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {}", e))?; - } - Ok(()) + let balance_streamer = EthBalanceEventStreamer::try_new(json!({}), self.clone()) + .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))?; + ctx.event_stream_manager + .add(balance_streamer, self.spawner().weak()) + .await + .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {}", e)) } /// Requests the nonce from all available nodes and returns the highest nonce available with the list of nodes that returned the highest nonce. diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 0744f34e85..79658d4e6c 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -6,7 +6,7 @@ use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Event, EventStreamer, EventName}; +use mm2_event_stream::{Event, EventName, EventStreamer, NoDataIn}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -148,10 +148,13 @@ async fn fetch_balance( #[async_trait] impl EventStreamer for EthBalanceEventStreamer { - fn event_name() -> EventName { EventName::BALANCE } + type DataInType = NoDataIn; - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker) } + + async fn handle(self, tx: oneshot::Sender>, _: impl StreamHandlerInput) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let streamer_id = self.streamer_id(); async fn start_polling(coin: EthCoin, ctx: MmArc, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { @@ -178,11 +181,7 @@ impl EventStreamer for EthBalanceEventStreamer { log::error!("Failed getting addresses for {}. Error: {}", coin.ticker, e); let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); ctx.stream_channel_controller - .broadcast(Event::err( - format!("{}:{}", EthBalanceEventStreamer::event_name(), coin.ticker), - e, - None, - )) + .broadcast(Event::err(streamer_id.clone(), e, None)) .await; sleep_remaining_time(interval, now).await; continue; @@ -215,17 +214,9 @@ impl EventStreamer for EthBalanceEventStreamer { err.error ); let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); + // FIXME: We should add the address in the error message. ctx.stream_channel_controller - .broadcast(Event::err( - format!( - "{}:{}:{}", - EthBalanceEventStreamer::event_name(), - err.ticker, - err.address - ), - e, - None, - )) + .broadcast(Event::err(streamer_id.clone(), e, None)) .await; }, }; @@ -258,30 +249,4 @@ impl EventStreamer for EthBalanceEventStreamer { start_polling(self.coin, ctx, self.interval).await } - - async fn spawn(self) -> Result<(), String> { - if !self.enabled { - return Ok(()); - } - log::info!( - "{} event is activated for {}, and polling every {}s", - Self::event_name(), - self.coin.ticker, - self.interval - ); - - let (tx, rx) = oneshot::channel(); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.coin.ticker - )); - self.coin.spawner().spawn_with_settings(self.handle(tx), settings); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler was aborted before sending event initialization status: {e}" - )) - }) - } } diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index c104dfe356..55e5bdf176 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -6,7 +6,7 @@ use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, EventName}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; @@ -35,10 +35,13 @@ impl TendermintBalanceEventStreamer { #[async_trait] impl EventStreamer for TendermintBalanceEventStreamer { - fn event_name() -> EventName { EventName::BALANCE } + type DataInType = NoDataIn; - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } + + async fn handle(self, tx: oneshot::Sender>, _: impl StreamHandlerInput) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let streamer_id = self.streaemr_id(); let coin = self.coin; fn generate_subscription_query(query_filter: String) -> String { @@ -143,7 +146,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); ctx.stream_channel_controller - .broadcast(Event::err(format!("{}:{}", Self::event_name(), ticker), e, None)) + .broadcast(Event::err(streamer_id.clone(), e, None)) .await; continue; @@ -175,32 +178,11 @@ impl EventStreamer for TendermintBalanceEventStreamer { if !balance_updates.is_empty() { ctx.stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), json!(balance_updates), None)) + .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) .await; } } } } } - - async fn spawn(self) -> Result<(), String> { - if !self.enabled { - return Ok(()); - } - log::info!("{} event is activated for {}.", Self::event_name(), self.coin.ticker(),); - - let (tx, rx) = oneshot::channel(); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.coin.ticker() - )); - self.coin.spawner().spawn_with_settings(self.handle(tx), settings); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler was aborted before sending event initialization status: {e}" - )) - }) - } } diff --git a/mm2src/coins/tendermint/tendermint_coin.rs b/mm2src/coins/tendermint/tendermint_coin.rs index 42382c06e6..1b1eb37d82 100644 --- a/mm2src/coins/tendermint/tendermint_coin.rs +++ b/mm2src/coins/tendermint/tendermint_coin.rs @@ -350,7 +350,7 @@ pub struct TendermintCoinImpl { pub(crate) history_sync_state: Mutex, client: TendermintRpcClient, pub(crate) chain_registry_name: Option, - pub(crate) ctx: MmWeak, + pub ctx: MmWeak, pub(crate) is_keplr_from_ledger: bool, } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 5978b4c092..d02ddf0662 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -5,7 +5,7 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, EventName, Filter}; +use mm2_event_stream::{Event, EventName, EventStreamer, Filter}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -48,17 +48,24 @@ impl UtxoBalanceEventStreamer { enabled, // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. // The UtxoArc might belong to a different coin type though. - coin: UtxoStandardCoin::from(utxo_arc), + coin: UtxoStandardCoin::from(utxo_arc).spawner(), }) } } #[async_trait] impl EventStreamer for UtxoBalanceEventStreamer { - fn event_name() -> EventName { EventName::BALANCE } + type DataInType = ScripthashNotification; - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } + + async fn handle( + self, + tx: oneshot::Sender>, + data_rx: impl StreamHandlerInput, + ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let streamer_id = self.streamer_id(); let coin = self.coin; let filter = Arc::new(UtxoBalanceEventFilter::new(coin.ticker())); @@ -99,6 +106,13 @@ impl EventStreamer for UtxoBalanceEventStreamer { Ok(scripthash_to_address_map) } + if coin.as_ref().rpc_client.is_native() { + log::warn!("Native RPC client is not supported for {streamer_id} event. Skipping event initialization."); + // We won't consider this an error but just an unsupported scenario and continue running. + tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + panic!("Native RPC client is not supported for UtxoBalanceEventStreamer."); + } + let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { Some(ctx) => ctx, None => { @@ -131,7 +145,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", Self::event_name(), coin.ticker()), + streamer_id.clone(), json!({ "error": e }), Some(filter.clone()), )) @@ -150,7 +164,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::err( - format!("{}:{}", Self::event_name(), coin.ticker()), + streamer_id.clone(), json!({ "error": e }), Some(filter.clone()), )) @@ -204,13 +218,8 @@ impl EventStreamer for UtxoBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); - // FIXME: Note that such an event isn't SSE-ed to any client since no body is listening to it. ctx.stream_channel_controller - .broadcast(Event::err( - format!("{}:{}", Self::event_name(), ticker), - e, - Some(filter.clone()), - )) + .broadcast(Event::err(streamer_id.clone(), e, Some(filter.clone()))) .await; continue; @@ -225,48 +234,13 @@ impl EventStreamer for UtxoBalanceEventStreamer { ctx.stream_channel_controller .broadcast(Event::new( - Self::event_name().to_string(), + streamer_id.clone(), json!(vec![payload]), Some(filter.clone()), )) .await; } } - - async fn spawn(self) -> Result<(), String> { - if !self.enabled { - return Ok(()); - } - - if self.coin.as_ref().rpc_client.is_native() { - log::warn!( - "Native RPC client is not supported for {} event. Skipping event initialization.", - self.event_id(), - ); - // We won't consider this an error but just an unsupported scenario and continue running. - return Ok(()); - } - - log::info!( - "{} event is activated for coin: {}", - Self::event_name(), - self.coin.ticker(), - ); - - let (tx, rx) = oneshot::channel(); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.coin.ticker() - )); - self.coin.spawner().spawn_with_settings(self.handle(tx), settings); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler was aborted before sending event initialization status: {e}" - )) - }) - } } struct UtxoBalanceEventFilter { @@ -278,7 +252,8 @@ impl UtxoBalanceEventFilter { pub fn new(ticker: &str) -> Self { Self { // The client requested event must have our ticker in that format to pass through. - event_name_match: format!("{}_{}", UtxoBalanceEventStreamer::event_name(), ticker), + // fixme: Abandon this filter matching + event_name_match: String::new(), //format!("BALANCE_{}", UtxoBalanceEventStreamer::event_name(), ticker), } } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index b6d65f86e6..f4af90fbd6 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -120,25 +120,16 @@ where spawn_block_header_utxo_loop(self.ticker, &utxo_arc, sync_handle, spv_conf); } - if let Some(config) = self - .ctx() - .event_stream_configuration - .get_event(&UtxoBalanceEventStreamer::event_name()) - { - UtxoBalanceEventStreamer::try_new(config, utxo_arc) - .map_to_mm(|e| { - UtxoCoinBuildError::FailedSpawningBalanceEvents(format!( - "Failed to initialize utxo event streaming: {e}" - )) - })? - .spawn() - .await - .map_to_mm(|e| { - UtxoCoinBuildError::FailedSpawningBalanceEvents(format!( - "Failed to spawn utxo event streaming: {e}" - )) - })?; - } + let utxo_streamer = UtxoBalanceEventStreamer::try_new(json!({}), utxo_arc.clone()).map_to_mm(|e| { + UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to initialize utxo event streaming: {e}")) + })?; + self.ctx + .event_stream_manager + .add(utxo_streamer, utxo_arc.abortable_system.weak_spawner()) + .await + .map_to_mm(|e| { + UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to spawn utxo event streaming: {e}")) + })?; Ok(result_coin) } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index ae16598b39..e0d026eb46 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -662,18 +662,12 @@ impl ZCoin { } async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - if let Some(config) = ctx - .event_stream_configuration - .get_event(&ZCoinBalanceEventStreamer::event_name()) - { - ZCoinBalanceEventStreamer::try_new(config, self.clone()) - .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))? - .spawn() - .await - .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {}", e))?; - } - - Ok(()) + let balance_streamer = ZCoinBalanceEventStreamer::try_new(json!({}), self.clone()) + .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))?; + ctx.event_stream_manager + .add(balance_streamer, self.spawner().weak()) + .await + .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {}", e)) } } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 64168bbfc3..9237cfd7a8 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -11,8 +11,7 @@ use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::EventStreamer; -use mm2_event_stream::{Event, EventName}; +use mm2_event_stream::{Event, EventStreamer, StreamerHandleInput}; use serde_json::Value as Json; use std::sync::Arc; @@ -39,10 +38,13 @@ impl ZCoinBalanceEventStreamer { #[async_trait] impl EventStreamer for ZCoinBalanceEventStreamer { - fn event_name() -> EventName { EventName::BALANCE } + type DataInType = (); - async fn handle(self, tx: oneshot::Sender>) { + fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } + + async fn handle(self, tx: oneshot::Sender>, data_rx: impl StreamerHandleInput<()>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + let streamer_id = self.streamer_id(); let coin = self.coin; macro_rules! send_status_on_err { @@ -82,7 +84,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { }); ctx.stream_channel_controller - .broadcast(Event::new(Self::event_name().to_string(), payload, None)) + .broadcast(Event::new(streamer_id.clone(), payload, None)) .await; }, Err(err) => { @@ -91,36 +93,10 @@ impl EventStreamer for ZCoinBalanceEventStreamer { let e = serde_json::to_value(err).expect("Serialization should't fail."); return ctx .stream_channel_controller - .broadcast(Event::err(format!("{}:{}", Self::event_name(), ticker), e, None)) + .broadcast(Event::err(streamer_id.clone(), e, None)) .await; }, }; } } - - async fn spawn(self) -> Result<(), String> { - if !self.enabled { - return Ok(()); - } - info!( - "{} event is activated for {} address {}.", - Self::event_name(), - self.coin.ticker(), - self.coin.my_z_address_encoded(), - ); - - let (tx, rx) = oneshot::channel(); - let settings = AbortSettings::info_on_abort(format!( - "{} event is stopped for {}.", - Self::event_name(), - self.coin.ticker() - )); - self.coin.spawner().spawn_with_settings(self.handle(tx), settings); - - rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler was aborted before sending event initialization status: {e}" - )) - }) - } } diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 57a7f45dd1..3dd2f77c04 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -21,12 +21,12 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_event_stream::EventStreamConfiguration; +use mm2_event_stream::EventStreamer; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value as Json; +use serde_json::{json, Value as Json}; use std::collections::{HashMap, HashSet}; impl TokenOf for TendermintToken { @@ -373,24 +373,28 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { &self, config: &EventStreamConfiguration, ) -> Result<(), MmError> { - if let Some(config) = config.get_event(&TendermintBalanceEventStreamer::event_name()) { - TendermintBalanceEventStreamer::try_new(config, self.clone()) - .map_to_mm(|e| TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::BalanceStreamInitError(format!( - "Failed to initialize tendermint balance streaming: {e}" - )), - })? - .spawn() - .await - .map_to_mm(|e| TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::BalanceStreamInitError(format!( - "Failed to spawn tendermint balance streaming handler: {e}" - )), - })?; - } - Ok(()) + let balance_streamer = + TendermintBalanceEventStreamer::try_new(json!({}), self.clone()).map_to_mm(|e| TendermintInitError { + ticker: self.ticker().to_owned(), + kind: TendermintInitErrorKind::BalanceStreamInitError(format!( + "Failed to initialize tendermint balance streaming: {e}" + )), + })?; + let ctx = MmArc::from_weak(&self.ctx) + .ok_or_else(|| TendermintInitError { + ticker: self.ticker().to_owned(), + kind: TendermintInitErrorKind::Internal("MM context must have been initialized already.".to_owned()), + }) + .map_to_mm(|e| e)?; + ctx.event_stream_manager + .add(balance_streamer, self.spawner().weak()) + .await + .map_to_mm(|e| TendermintInitError { + ticker: self.ticker().to_owned(), + kind: TendermintInitErrorKind::BalanceStreamInitError(format!( + "Failed to spawn tendermint balance streaming handler: {e}" + )), + }) } fn rpc_task_manager( diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 94516d17bd..338fdec74a 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -27,9 +27,9 @@ pub struct StreamingManager { impl StreamingManager { /// Spawns and adds a new streamer `streamer` to the manager. pub async fn add(&self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), String> { - let streamer_id = streamer.streamer_id().to_string(); + let streamer_id = streamer.streamer_id(); // NOTE: We spawn the streamer *before* checking if it can be added or not because - // we don't know how much time will it take for spawning and we don't want to lock + // we don't know how long will it take for to spawn up and we don't want to lock // the manager for too long. let channels = streamer.spawn(spawner).await?; let mut streamers = self.streamers.write().unwrap(); diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index e3db8a62c4..ff68033a3b 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -24,7 +24,7 @@ where /// Returns a human readable unique identifier for the event streamer. /// No other event streamer should have the same identifier. - fn streamer_id(&self) -> &str; + fn streamer_id(&self) -> String; /// Event handler that is responsible for broadcasting event data to the streaming channels. /// @@ -49,7 +49,7 @@ where // Also for AbortSettings, we can make this customizable with a default impl. spawner: WeakSpawner, ) -> Result<(oneshot::Sender<()>, Option>>), String> { - let streamer_id = self.streamer_id().to_string(); + let streamer_id = self.streamer_id(); info!("Spawning event streamer: {streamer_id}"); // A oneshot channel to receive the initialization status of the handler through. diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 6e0e84f2e1..02d67de26f 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -34,15 +34,15 @@ impl HeartbeatEvent { impl EventStreamer for HeartbeatEvent { type DataInType = NoDataIn; - fn streamer_id(&self) -> &str { "HEARTBEAT" } + fn streamer_id(&self) -> String { "HEARTBEAT".to_string() } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { tx.send(Ok(())).unwrap(); loop { self.ctx .stream_channel_controller - .broadcast(Event::new(self.streamer_id().to_string(), json!({}), None)) + .broadcast(Event::new(self.streamer_id(), json!({}), None)) .await; Timer::sleep(self.config.stream_interval_seconds).await; diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 6dfdd23d6c..72d0e063d5 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -430,6 +430,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { let network_steamer = NetworkEvent::try_new(config, ctx.clone()) .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))?; ctx.event_stream_manager + // FIXME: weak doesnt work, get rid of mmfutspawner & coinfutspawner and replace them with plain week spawner. .add(network_steamer, ctx.spawner().weak()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e}")))?; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index d93eb6bf8b..58d0987e41 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -39,9 +39,9 @@ impl NetworkEvent { impl EventStreamer for NetworkEvent { type DataInType = NoDataIn; - fn streamer_id(&self) -> &str { "NETWORK" } + fn streamer_id(&self) -> String { "NETWORK".to_string() } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { let p2p_ctx = P2PContext::fetch_from_mm_arc(&self.ctx); let mut previously_sent = json!({}); @@ -67,7 +67,7 @@ impl EventStreamer for NetworkEvent { if previously_sent != event_data || self.config.always_send { self.ctx .stream_channel_controller - .broadcast(Event::new(self.streamer_id().to_string(), event_data.clone(), None)) + .broadcast(Event::new(self.streamer_id(), event_data.clone(), None)) .await; previously_sent = event_data; From 700e9355f502c04bdbe5d2bd9899bf2dd55e4ea9 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 13:37:26 +0200 Subject: [PATCH 023/128] abandon CoinFutSpawner & MmFutSpawner in favor of WeakSpawner --- mm2src/coins/eth.rs | 38 ++++++++--------- mm2src/coins/lightning.rs | 9 ++-- mm2src/coins/lightning/ln_platform.rs | 4 +- mm2src/coins/lp_coins.rs | 41 +----------------- mm2src/coins/qrc20.rs | 31 +++++++------- mm2src/coins/sia.rs | 25 ++++++----- mm2src/coins/solana.rs | 26 ++++++------ mm2src/coins/solana/spl.rs | 16 +++---- mm2src/coins/tendermint/tendermint_coin.rs | 8 ++-- mm2src/coins/tendermint/tendermint_token.rs | 21 +++++----- mm2src/coins/test_coin.rs | 30 ++++++------- mm2src/coins/utxo.rs | 10 ++--- mm2src/coins/utxo/bch.rs | 2 +- mm2src/coins/utxo/qtum.rs | 2 +- mm2src/coins/utxo/slp.rs | 10 ++--- mm2src/coins/utxo/utxo_standard.rs | 2 +- mm2src/coins/z_coin.rs | 12 +++--- .../src/tendermint_with_assets_activation.rs | 2 +- .../src/utxo_activation/common_impl.rs | 4 +- mm2src/mm2_core/src/mm_ctx.rs | 42 +------------------ mm2src/mm2_main/src/lp_native_dex.rs | 10 ++--- 21 files changed, 136 insertions(+), 209 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 0b1bf6687b..67065d57fb 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -112,23 +112,23 @@ cfg_wasm32! { } use super::{coin_conf, lp_coinfind_or_err, AsyncMutex, BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, - CoinBalance, CoinFutSpawner, CoinProtocol, CoinTransportMetrics, CoinsContext, ConfirmPaymentInput, - EthValidateFeeArgs, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, IguanaPrivKey, MakerSwapTakerCoin, - MarketCoinOps, MmCoin, MmCoinEnum, MyAddressError, MyWalletAddress, NegotiateSwapContractAddrErr, - NumConversError, NumConversResult, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, - PrivKeyBuildPolicy, PrivKeyPolicyNotAllowed, RawTransactionError, RawTransactionFut, - RawTransactionRequest, RawTransactionRes, RawTransactionResult, RefundError, RefundPaymentArgs, - RefundResult, RewardTarget, RpcClientType, RpcTransportEventHandler, RpcTransportEventHandlerShared, - SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignEthTransactionParams, - SignRawTransactionEnum, SignRawTransactionRequest, SignatureError, SignatureResult, SpendPaymentArgs, - SwapOps, SwapTxFeePolicy, TakerSwapMakerCoin, TradeFee, TradePreimageError, TradePreimageFut, - TradePreimageResult, TradePreimageValue, Transaction, TransactionDetails, TransactionEnum, TransactionErr, - TransactionFut, TransactionType, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, - ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, - ValidatePaymentFut, ValidatePaymentInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, - WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, - WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WithdrawError, WithdrawFee, WithdrawFut, - WithdrawRequest, WithdrawResult, EARLY_CONFIRMATION_ERR_LOG, INVALID_CONTRACT_ADDRESS_ERR_LOG, + CoinBalance, CoinProtocol, CoinTransportMetrics, CoinsContext, ConfirmPaymentInput, EthValidateFeeArgs, + FeeApproxStage, FoundSwapTxSpend, HistorySyncState, IguanaPrivKey, MakerSwapTakerCoin, MarketCoinOps, + MmCoin, MmCoinEnum, MyAddressError, MyWalletAddress, NegotiateSwapContractAddrErr, NumConversError, + NumConversResult, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, + PrivKeyPolicyNotAllowed, RawTransactionError, RawTransactionFut, RawTransactionRequest, RawTransactionRes, + RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, RewardTarget, RpcClientType, + RpcTransportEventHandler, RpcTransportEventHandlerShared, SearchForSwapTxSpendInput, + SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignEthTransactionParams, SignRawTransactionEnum, + SignRawTransactionRequest, SignatureError, SignatureResult, SpendPaymentArgs, SwapOps, SwapTxFeePolicy, + TakerSwapMakerCoin, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, + TradePreimageValue, Transaction, TransactionDetails, TransactionEnum, TransactionErr, TransactionFut, + TransactionType, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, + ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, + ValidatePaymentInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, + WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest, + WithdrawResult, EARLY_CONFIRMATION_ERR_LOG, INVALID_CONTRACT_ADDRESS_ERR_LOG, INVALID_PAYMENT_STATE_ERR_LOG, INVALID_RECEIVER_ERR_LOG, INVALID_SENDER_ERR_LOG, INVALID_SWAP_ID_ERR_LOG}; pub use rlp; cfg_native! { @@ -5286,7 +5286,7 @@ impl EthCoin { let balance_streamer = EthBalanceEventStreamer::try_new(json!({}), self.clone()) .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))?; ctx.event_stream_manager - .add(balance_streamer, self.spawner().weak()) + .add(balance_streamer, self.spawner()) .await .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {}", e)) } @@ -5417,7 +5417,7 @@ impl EthTxFeeDetails { impl MmCoin for EthCoin { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { Box::new(get_raw_transaction_impl(self.clone(), req).boxed().compat()) diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 2feb7ef014..1f8952c94a 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -15,8 +15,8 @@ use crate::lightning::ln_utils::{filter_channels, pay_invoice_with_max_total_clt use crate::utxo::rpc_clients::UtxoRpcClientEnum; use crate::utxo::utxo_common::{big_decimal_from_sat, big_decimal_from_sat_unsigned}; use crate::utxo::{sat_from_big_decimal, utxo_common, BlockchainNetwork}; -use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, DexFee, - FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, +use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DexFee, FeeApproxStage, + FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, RawTransactionError, RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, @@ -27,7 +27,8 @@ use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, C ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, ValidateWatcherSpendInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, - WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WithdrawError, WithdrawFut, WithdrawRequest}; + WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFut, + WithdrawRequest}; use async_trait::async_trait; use bitcoin::bech32::ToBase32; use bitcoin::hashes::Hash; @@ -1274,7 +1275,7 @@ struct LightningProtocolInfo { impl MmCoin for LightningCoin { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.platform.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.platform.abortable_system.weak_spawner() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { let fut = async move { diff --git a/mm2src/coins/lightning/ln_platform.rs b/mm2src/coins/lightning/ln_platform.rs index 4d0573af95..8a7ebe191c 100644 --- a/mm2src/coins/lightning/ln_platform.rs +++ b/mm2src/coins/lightning/ln_platform.rs @@ -6,7 +6,7 @@ use crate::utxo::rpc_clients::{BestBlock as RpcBestBlock, BlockHashOrHeight, Con use crate::utxo::spv::SimplePaymentVerification; use crate::utxo::utxo_standard::UtxoStandardCoin; use crate::utxo::GetConfirmedTxError; -use crate::{CoinFutSpawner, MarketCoinOps, MmCoin, WaitForHTLCTxSpendArgs}; +use crate::{MarketCoinOps, MmCoin, WaitForHTLCTxSpendArgs, WeakSpawner}; use bitcoin::blockdata::block::BlockHeader; use bitcoin::blockdata::script::Script; use bitcoin::blockdata::transaction::Transaction; @@ -216,7 +216,7 @@ impl Platform { #[inline] fn rpc_client(&self) -> &UtxoRpcClientEnum { &self.coin.as_ref().rpc_client } - pub fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.abortable_system) } + pub fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } pub async fn set_latest_fees(&self) -> UtxoRpcResult<()> { let platform_coin = &self.coin; diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 303b60d9f9..356503b6f2 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -3228,8 +3228,8 @@ pub trait MmCoin: /// /// # Note /// - /// `CoinFutSpawner` doesn't prevent the spawned futures from being aborted. - fn spawner(&self) -> CoinFutSpawner; + /// `WeakSpawner` doesn't prevent the spawned futures from being aborted. + fn spawner(&self) -> WeakSpawner; fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut; @@ -3355,43 +3355,6 @@ pub trait MmCoin: fn on_token_deactivated(&self, ticker: &str); } -/// The coin futures spawner. It's used to spawn futures that can be aborted immediately or after a timeout -/// on the the coin deactivation. -/// -/// # Note -/// -/// `CoinFutSpawner` doesn't prevent the spawned futures from being aborted. -#[derive(Clone)] -pub struct CoinFutSpawner { - inner: WeakSpawner, -} - -impl CoinFutSpawner { - pub fn new(system: &AbortableQueue) -> CoinFutSpawner { - CoinFutSpawner { - inner: system.weak_spawner(), - } - } -} - -impl SpawnFuture for CoinFutSpawner { - fn spawn(&self, f: F) - where - F: Future03 + Send + 'static, - { - self.inner.spawn(f) - } -} - -impl SpawnAbortable for CoinFutSpawner { - fn spawn_with_settings(&self, fut: F, settings: AbortSettings) - where - F: Future03 + Send + 'static, - { - self.inner.spawn_with_settings(fut, settings) - } -} - #[derive(Clone)] #[allow(clippy::large_enum_variant)] pub enum MmCoinEnum { diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 3ee9e7761b..28f1a90748 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -16,20 +16,21 @@ use crate::utxo::{qtum, ActualTxFee, AdditionalTxData, AddrFromStrError, Broadca UnsupportedAddr, UtxoActivationParams, UtxoAddressFormat, UtxoCoinFields, UtxoCommonOps, UtxoFromLegacyReqErr, UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps, VerboseTransactionFrom, UTXO_LOCK}; -use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, - DexFee, Eip1559Ops, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, IguanaPrivKey, MakerSwapTakerCoin, - MarketCoinOps, MmCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, - PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, PrivKeyPolicyNotAllowed, - RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, - RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, - SignRawTransactionRequest, SignatureResult, SpendPaymentArgs, SwapOps, SwapTxFeePolicy, - TakerSwapMakerCoin, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, - TradePreimageValue, TransactionData, TransactionDetails, TransactionEnum, TransactionErr, TransactionFut, - TransactionResult, TransactionType, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, - ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentFut, - ValidatePaymentInput, ValidateWatcherSpendInput, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, - WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, - WatcherValidateTakerFeeInput, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest, WithdrawResult}; +use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DexFee, Eip1559Ops, + FeeApproxStage, FoundSwapTxSpend, HistorySyncState, IguanaPrivKey, MakerSwapTakerCoin, MarketCoinOps, + MmCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, PaymentInstructions, + PaymentInstructionsErr, PrivKeyBuildPolicy, PrivKeyPolicyNotAllowed, RawTransactionFut, + RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, + SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, + SignatureResult, SpendPaymentArgs, SwapOps, SwapTxFeePolicy, TakerSwapMakerCoin, TradeFee, + TradePreimageError, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionData, + TransactionDetails, TransactionEnum, TransactionErr, TransactionFut, TransactionResult, TransactionType, + TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, + ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentFut, ValidatePaymentInput, + ValidateWatcherSpendInput, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, + WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest, + WithdrawResult}; use async_trait::async_trait; use bitcrypto::{dhash160, sha256}; use chain::TransactionOutput; @@ -1312,7 +1313,7 @@ impl MarketCoinOps for Qrc20Coin { impl MmCoin for Qrc20Coin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo) } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.as_ref().abortable_system) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { Box::new(qrc20_withdraw(self.clone(), req).boxed().compat()) diff --git a/mm2src/coins/sia.rs b/mm2src/coins/sia.rs index 446a507070..1dfefb6369 100644 --- a/mm2src/coins/sia.rs +++ b/mm2src/coins/sia.rs @@ -1,17 +1,16 @@ use super::{CoinBalance, HistorySyncState, MarketCoinOps, MmCoin, RawTransactionFut, RawTransactionRequest, SwapOps, TradeFee, TransactionEnum, TransactionFut}; -use crate::{coin_errors::MyAddressError, BalanceFut, CanRefundHtlc, CheckIfMyPaymentSentArgs, CoinFutSpawner, - ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, MakerSwapTakerCoin, MmCoinEnum, - NegotiateSwapContractAddrErr, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, - PrivKeyBuildPolicy, PrivKeyPolicy, RawTransactionResult, RefundPaymentArgs, RefundResult, - SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, - SignatureResult, SpendPaymentArgs, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, - TradePreimageValue, TransactionResult, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, - ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, - ValidatePaymentFut, ValidatePaymentInput, ValidatePaymentResult, ValidateWatcherSpendInput, - VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, - WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WithdrawFut, - WithdrawRequest}; +use crate::{coin_errors::MyAddressError, BalanceFut, CanRefundHtlc, CheckIfMyPaymentSentArgs, ConfirmPaymentInput, + DexFee, FeeApproxStage, FoundSwapTxSpend, MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, + PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, PrivKeyPolicy, + RawTransactionResult, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, + SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, SignatureResult, + SpendPaymentArgs, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, TradePreimageValue, + TransactionResult, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, + ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, + ValidatePaymentInput, ValidatePaymentResult, ValidateWatcherSpendInput, VerificationResult, + WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, + WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WeakSpawner, WithdrawFut, WithdrawRequest}; use async_trait::async_trait; use common::executor::AbortedError; use futures::{FutureExt, TryFutureExt}; @@ -214,7 +213,7 @@ pub struct SiaCoinProtocolInfo; impl MmCoin for SiaCoin { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { unimplemented!() } + fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/solana.rs b/mm2src/coins/solana.rs index 2b9abef1ca..4507d1a245 100644 --- a/mm2src/coins/solana.rs +++ b/mm2src/coins/solana.rs @@ -44,18 +44,18 @@ use crate::coin_errors::{MyAddressError, ValidatePaymentResult}; use crate::hd_wallet::HDPathAccountToAddressId; use crate::solana::{solana_common::{lamports_to_sol, PrepareTransferData, SufficientBalanceError}, spl::SplTokenInfo}; -use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinFutSpawner, ConfirmPaymentInput, DexFee, - FeeApproxStage, FoundSwapTxSpend, MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, - PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, - PrivKeyPolicyNotAllowed, RawTransactionError, RawTransactionFut, RawTransactionRequest, - RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, - SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, SignatureResult, - SpendPaymentArgs, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, TradePreimageValue, - TransactionData, TransactionDetails, TransactionFut, TransactionResult, TransactionType, TxMarshalingErr, - UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, - ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, - ValidateWatcherSpendInput, VerificationResult, WaitForHTLCTxSpendArgs, WatcherReward, WatcherRewardError, - WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, +use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, ConfirmPaymentInput, DexFee, FeeApproxStage, + FoundSwapTxSpend, MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, + PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, PrivKeyPolicyNotAllowed, + RawTransactionError, RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, + RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, + SendPaymentArgs, SignRawTransactionRequest, SignatureResult, SpendPaymentArgs, TakerSwapMakerCoin, + TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionData, TransactionDetails, + TransactionFut, TransactionResult, TransactionType, TxMarshalingErr, UnexpectedDerivationMethod, + ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, + ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, ValidateWatcherSpendInput, + VerificationResult, WaitForHTLCTxSpendArgs, WatcherReward, WatcherRewardError, + WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFut, WithdrawRequest, WithdrawResult}; pub mod solana_common; @@ -997,7 +997,7 @@ impl WatcherOps for SolanaCoin { impl MmCoin for SolanaCoin { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { Box::new(Box::pin(withdraw_impl(self.clone(), req)).compat()) diff --git a/mm2src/coins/solana/spl.rs b/mm2src/coins/solana/spl.rs index 7bc720e5e7..64baf8438b 100644 --- a/mm2src/coins/solana/spl.rs +++ b/mm2src/coins/solana/spl.rs @@ -2,18 +2,18 @@ use super::{CoinBalance, HistorySyncState, MarketCoinOps, MmCoin, SwapOps, Trade use crate::coin_errors::{MyAddressError, ValidatePaymentResult}; use crate::solana::solana_common::{ui_amount_to_amount, PrepareTransferData, SufficientBalanceError}; use crate::solana::{solana_common, AccountError, SolanaCommonOps, SolanaFeeDetails}; -use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinFutSpawner, ConfirmPaymentInput, DexFee, FeeApproxStage, - FoundSwapTxSpend, MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, - PaymentInstructions, PaymentInstructionsErr, RawTransactionError, RawTransactionFut, - RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, - SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, - SignatureResult, SolanaCoin, SpendPaymentArgs, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, +use crate::{BalanceFut, CheckIfMyPaymentSentArgs, ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, + MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, PaymentInstructions, + PaymentInstructionsErr, RawTransactionError, RawTransactionFut, RawTransactionRequest, + RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, + SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, SignatureResult, + SolanaCoin, SpendPaymentArgs, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionData, TransactionDetails, TransactionFut, TransactionResult, TransactionType, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, ValidateWatcherSpendInput, VerificationResult, WaitForHTLCTxSpendArgs, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, - WatcherValidateTakerFeeInput, WithdrawError, WithdrawFut, WithdrawRequest, WithdrawResult}; + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFut, WithdrawRequest, WithdrawResult}; use async_trait::async_trait; use bincode::serialize; use common::executor::{abortable_queue::AbortableQueue, AbortableSystem, AbortedError}; @@ -526,7 +526,7 @@ impl WatcherOps for SplToken { impl MmCoin for SplToken { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.conf.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.conf.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { Box::new(Box::pin(withdraw_impl(self.clone(), req)).compat()) diff --git a/mm2src/coins/tendermint/tendermint_coin.rs b/mm2src/coins/tendermint/tendermint_coin.rs index 1b1eb37d82..2eccceffe5 100644 --- a/mm2src/coins/tendermint/tendermint_coin.rs +++ b/mm2src/coins/tendermint/tendermint_coin.rs @@ -14,8 +14,8 @@ use crate::tendermint::ibc::IBC_OUT_SOURCE_PORT; use crate::utxo::sat_from_big_decimal; use crate::utxo::utxo_common::big_decimal_from_sat; use crate::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BigDecimal, CheckIfMyPaymentSentArgs, - CoinBalance, CoinFutSpawner, ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, - HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, NegotiateSwapContractAddrErr, + CoinBalance, ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, + MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, NegotiateSwapContractAddrErr, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyBuildPolicy, PrivKeyPolicy, PrivKeyPolicyNotAllowed, RawTransactionError, RawTransactionFut, RawTransactionRequest, RawTransactionRes, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, RpcCommonOps, @@ -27,7 +27,7 @@ use crate::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BigDecimal, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentFut, ValidatePaymentInput, ValidateWatcherSpendInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, - WatcherValidateTakerFeeInput, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest}; + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest}; use async_std::prelude::FutureExt as AsyncStdFutureExt; use async_trait::async_trait; use bip32::DerivationPath; @@ -2128,7 +2128,7 @@ impl MmCoin for TendermintCoin { wallet_only_conf || self.is_keplr_from_ledger } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { let coin = self.clone(); diff --git a/mm2src/coins/tendermint/tendermint_token.rs b/mm2src/coins/tendermint/tendermint_token.rs index 90407fa8ae..cc7b99ddf4 100644 --- a/mm2src/coins/tendermint/tendermint_token.rs +++ b/mm2src/coins/tendermint/tendermint_token.rs @@ -6,18 +6,19 @@ use super::{create_withdraw_msg_as_any, TendermintCoin, TendermintFeeDetails, GA use crate::coin_errors::ValidatePaymentResult; use crate::utxo::utxo_common::big_decimal_from_sat; use crate::{big_decimal_from_sat_unsigned, utxo::sat_from_big_decimal, BalanceFut, BigDecimal, - CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, FeeApproxStage, - FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MyAddressError, - NegotiateSwapContractAddrErr, PaymentInstructions, PaymentInstructionsErr, RawTransactionError, - RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, - RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, - SignRawTransactionRequest, SignatureResult, SpendPaymentArgs, SwapOps, TakerSwapMakerCoin, TradeFee, - TradePreimageFut, TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, - TransactionErr, TransactionFut, TransactionResult, TransactionType, TxFeeDetails, TxMarshalingErr, + CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, FeeApproxStage, FoundSwapTxSpend, + HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MyAddressError, NegotiateSwapContractAddrErr, + PaymentInstructions, PaymentInstructionsErr, RawTransactionError, RawTransactionFut, + RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, + SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, + SignatureResult, SpendPaymentArgs, SwapOps, TakerSwapMakerCoin, TradeFee, TradePreimageFut, + TradePreimageResult, TradePreimageValue, TransactionDetails, TransactionEnum, TransactionErr, + TransactionFut, TransactionResult, TransactionType, TxFeeDetails, TxMarshalingErr, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherSearchForSwapTxSpendInput, - WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WithdrawError, WithdrawFut, WithdrawRequest}; + WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFut, + WithdrawRequest}; use crate::{DexFee, MmCoinEnum, PaymentInstructionArgs, ValidateWatcherSpendInput, WatcherReward, WatcherRewardError}; use async_trait::async_trait; use bitcrypto::sha256; @@ -479,7 +480,7 @@ impl MmCoin for TendermintToken { wallet_only_conf || self.platform_coin.is_keplr_from_ledger } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { let platform = self.platform_coin.clone(); diff --git a/mm2src/coins/test_coin.rs b/mm2src/coins/test_coin.rs index c077bf60bd..496c21fde2 100644 --- a/mm2src/coins/test_coin.rs +++ b/mm2src/coins/test_coin.rs @@ -4,20 +4,20 @@ use super::{CoinBalance, FundingTxSpend, HistorySyncState, MarketCoinOps, MmCoin RawTransactionRequest, SearchForFundingSpendErr, SwapOps, TradeFee, TransactionEnum, TransactionFut, WaitForTakerPaymentSpendError}; use crate::coin_errors::ValidatePaymentResult; -use crate::{coin_errors::MyAddressError, BalanceFut, CanRefundHtlc, CheckIfMyPaymentSentArgs, CoinFutSpawner, - ConfirmPaymentInput, FeeApproxStage, FoundSwapTxSpend, GenPreimageResult, GenTakerFundingSpendArgs, - GenTakerPaymentSpendArgs, MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, - ParseCoinAssocTypes, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, - RawTransactionResult, RefundFundingSecretArgs, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, - SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SendTakerFundingArgs, SignRawTransactionRequest, - SignatureResult, SpendPaymentArgs, TakerCoinSwapOpsV2, TakerSwapMakerCoin, TradePreimageFut, - TradePreimageResult, TradePreimageValue, Transaction, TransactionErr, TransactionResult, TxMarshalingErr, - TxPreimageWithSig, UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, - ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, - ValidatePaymentInput, ValidateSwapV2TxResult, ValidateTakerFundingArgs, - ValidateTakerFundingSpendPreimageResult, ValidateTakerPaymentSpendPreimageResult, VerificationResult, - WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, - WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WithdrawFut, WithdrawRequest}; +use crate::{coin_errors::MyAddressError, BalanceFut, CanRefundHtlc, CheckIfMyPaymentSentArgs, ConfirmPaymentInput, + FeeApproxStage, FoundSwapTxSpend, GenPreimageResult, GenTakerFundingSpendArgs, GenTakerPaymentSpendArgs, + MakerSwapTakerCoin, MmCoinEnum, NegotiateSwapContractAddrErr, ParseCoinAssocTypes, PaymentInstructionArgs, + PaymentInstructions, PaymentInstructionsErr, RawTransactionResult, RefundFundingSecretArgs, + RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, + SendPaymentArgs, SendTakerFundingArgs, SignRawTransactionRequest, SignatureResult, SpendPaymentArgs, + TakerCoinSwapOpsV2, TakerSwapMakerCoin, TradePreimageFut, TradePreimageResult, TradePreimageValue, + Transaction, TransactionErr, TransactionResult, TxMarshalingErr, TxPreimageWithSig, + UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, + ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, + ValidateSwapV2TxResult, ValidateTakerFundingArgs, ValidateTakerFundingSpendPreimageResult, + ValidateTakerPaymentSpendPreimageResult, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, + WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawFut, WithdrawRequest}; use crate::{DexFee, ToBytes, ValidateWatcherSpendInput}; use async_trait::async_trait; use common::executor::AbortedError; @@ -339,7 +339,7 @@ impl WatcherOps for TestCoin { impl MmCoin for TestCoin { fn is_asset_chain(&self) -> bool { unimplemented!() } - fn spawner(&self) -> CoinFutSpawner { unimplemented!() } + fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index ebd7f72f29..da5e640e23 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -102,13 +102,13 @@ use utxo_signer::{TxProvider, TxProviderError, UtxoSignTxError, UtxoSignTxResult use self::rpc_clients::{electrum_script_hash, ElectrumClient, ElectrumRpcRequest, EstimateFeeMethod, EstimateFeeMode, NativeClient, UnspentInfo, UnspentMap, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, UtxoRpcResult}; -use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResult, CoinBalance, CoinFutSpawner, - CoinsContext, DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, - MarketCoinOps, MmCoin, NumConversError, NumConversResult, PrivKeyActivationPolicy, PrivKeyPolicy, +use super::{big_decimal_from_sat_unsigned, BalanceError, BalanceFut, BalanceResult, CoinBalance, CoinsContext, + DerivationMethod, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, KmdRewardsDetails, MarketCoinOps, + MmCoin, NumConversError, NumConversResult, PrivKeyActivationPolicy, PrivKeyPolicy, PrivKeyPolicyNotAllowed, RawTransactionFut, RpcTransportEventHandler, RpcTransportEventHandlerShared, TradeFee, TradePreimageError, TradePreimageFut, TradePreimageResult, Transaction, TransactionDetails, - TransactionEnum, TransactionErr, UnexpectedDerivationMethod, VerificationError, WithdrawError, - WithdrawRequest}; + TransactionEnum, TransactionErr, UnexpectedDerivationMethod, VerificationError, WeakSpawner, + WithdrawError, WithdrawRequest}; use crate::coin_balance::{EnableCoinScanPolicy, EnabledCoinBalanceParams, HDAddressBalanceScanner}; use crate::hd_wallet::{HDAccountOps, HDAddressOps, HDPathAccountToAddressId, HDWalletCoinOps, HDWalletOps, HDWalletStorageError}; diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 2b94135933..60a3ae126a 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -1261,7 +1261,7 @@ impl MarketCoinOps for BchCoin { impl MmCoin for BchCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.as_ref().abortable_system) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { Box::new(utxo_common::get_raw_transaction(&self.utxo_arc, req).boxed().compat()) diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 8b8a60d246..44bf389459 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -880,7 +880,7 @@ impl MarketCoinOps for QtumCoin { impl MmCoin for QtumCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.as_ref().abortable_system) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { Box::new(utxo_common::get_raw_transaction(&self.utxo_arc, req).boxed().compat()) diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index c559449c22..0580d8dbdc 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -13,9 +13,9 @@ use crate::utxo::utxo_common::{self, big_decimal_from_sat_unsigned, payment_scri use crate::utxo::{generate_and_send_tx, sat_from_big_decimal, ActualTxFee, AdditionalTxData, BroadcastTxErr, FeePolicy, GenerateTxError, RecentlySpentOutPointsGuard, UtxoCoinConf, UtxoCoinFields, UtxoCommonOps, UtxoTx, UtxoTxBroadcastOps, UtxoTxGenerationOps}; -use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, DerivationMethod, - DexFee, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, - MmCoinEnum, NegotiateSwapContractAddrErr, NumConversError, PaymentInstructionArgs, PaymentInstructions, +use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DerivationMethod, DexFee, + FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, + NegotiateSwapContractAddrErr, NumConversError, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyPolicyNotAllowed, RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, SignRawTransactionRequest, SignatureResult, @@ -25,7 +25,7 @@ use crate::{BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, C UnexpectedDerivationMethod, ValidateAddressResult, ValidateFeeArgs, ValidateInstructionsErr, ValidateOtherPubKeyErr, ValidatePaymentInput, ValidateWatcherSpendInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, - WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, + WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFee, WithdrawFut, WithdrawRequest}; use async_trait::async_trait; use base64::engine::general_purpose::STANDARD; @@ -1597,7 +1597,7 @@ impl From for TxFeeDetails { impl MmCoin for SlpToken { fn is_asset_chain(&self) -> bool { false } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.conf.abortable_system) } + fn spawner(&self) -> WeakSpawner { self.conf.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { Box::new( diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 03c889d790..1cfd1aaf6e 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -909,7 +909,7 @@ impl MarketCoinOps for UtxoStandardCoin { impl MmCoin for UtxoStandardCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.as_ref().abortable_system) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { Box::new(utxo_common::get_raw_transaction(&self.utxo_arc, req).boxed().compat()) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index e0d026eb46..369a9a9319 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -27,9 +27,9 @@ use crate::z_coin::storage::{BlockDbImpl, WalletDbShared}; use crate::z_coin::z_balance_streaming::ZBalanceEventHandler; use crate::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; use crate::z_coin::z_tx_history::{fetch_tx_history_from_db, ZCoinTxHistoryItem}; -use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, CoinFutSpawner, ConfirmPaymentInput, - DexFee, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, - MmCoinEnum, NegotiateSwapContractAddrErr, NumConversError, PaymentInstructionArgs, PaymentInstructions, +use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DexFee, + FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, + NegotiateSwapContractAddrErr, NumConversError, PaymentInstructionArgs, PaymentInstructions, PaymentInstructionsErr, PrivKeyActivationPolicy, PrivKeyBuildPolicy, PrivKeyPolicyNotAllowed, RawTransactionFut, RawTransactionRequest, RawTransactionResult, RefundError, RefundPaymentArgs, RefundResult, SearchForSwapTxSpendInput, SendMakerPaymentSpendPreimageInput, SendPaymentArgs, @@ -40,7 +40,7 @@ use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, Coi ValidateOtherPubKeyErr, ValidatePaymentError, ValidatePaymentFut, ValidatePaymentInput, ValidateWatcherSpendInput, VerificationError, VerificationResult, WaitForHTLCTxSpendArgs, WatcherOps, WatcherReward, WatcherRewardError, WatcherSearchForSwapTxSpendInput, WatcherValidatePaymentInput, - WatcherValidateTakerFeeInput, WithdrawError, WithdrawFut, WithdrawRequest}; + WatcherValidateTakerFeeInput, WeakSpawner, WithdrawError, WithdrawFut, WithdrawRequest}; use async_trait::async_trait; use bitcrypto::dhash256; @@ -665,7 +665,7 @@ impl ZCoin { let balance_streamer = ZCoinBalanceEventStreamer::try_new(json!({}), self.clone()) .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))?; ctx.event_stream_manager - .add(balance_streamer, self.spawner().weak()) + .add(balance_streamer, self.spawner()) .await .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {}", e)) } @@ -1662,7 +1662,7 @@ impl WatcherOps for ZCoin { impl MmCoin for ZCoin { fn is_asset_chain(&self) -> bool { self.utxo_arc.conf.asset_chain } - fn spawner(&self) -> CoinFutSpawner { CoinFutSpawner::new(&self.as_ref().abortable_system) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, _req: WithdrawRequest) -> WithdrawFut { Box::new(futures01::future::err(MmError::new(WithdrawError::InternalError( diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 3dd2f77c04..1b1605b006 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -387,7 +387,7 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { }) .map_to_mm(|e| e)?; ctx.event_stream_manager - .add(balance_streamer, self.spawner().weak()) + .add(balance_streamer, self.spawner()) .await .map_to_mm(|e| TendermintInitError { ticker: self.ticker().to_owned(), diff --git a/mm2src/coins_activation/src/utxo_activation/common_impl.rs b/mm2src/coins_activation/src/utxo_activation/common_impl.rs index 8e3d071025..e22428264a 100644 --- a/mm2src/coins_activation/src/utxo_activation/common_impl.rs +++ b/mm2src/coins_activation/src/utxo_activation/common_impl.rs @@ -8,7 +8,7 @@ use coins::hd_wallet::RpcTaskXPubExtractor; use coins::my_tx_history_v2::TxHistoryStorage; use coins::utxo::utxo_tx_history_v2::{utxo_history_loop, UtxoTxHistoryOps}; use coins::utxo::{UtxoActivationParams, UtxoCoinFields}; -use coins::{CoinBalance, CoinFutSpawner, MarketCoinOps, PrivKeyActivationPolicy, PrivKeyBuildPolicy}; +use coins::{CoinBalance, MarketCoinOps, PrivKeyActivationPolicy, PrivKeyBuildPolicy, WeakSpawner}; use common::executor::{AbortSettings, SpawnAbortable}; use crypto::hw_rpc_task::HwConnectStatuses; use crypto::{CryptoCtxError, HwRpcError}; @@ -103,7 +103,7 @@ pub(crate) fn start_history_background_fetching( ) where Coin: AsRef + UtxoTxHistoryOps, { - let spawner = CoinFutSpawner::new(&coin.as_ref().abortable_system); + let spawner = coin.as_ref().abortable_system.weak_spawner(); let msg = format!("'utxo_history_loop' has been aborted for {}", coin.ticker()); let fut = utxo_history_loop(coin, storage, metrics, current_balances); diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 66f9fcd0d1..c81d3c1592 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -339,8 +339,8 @@ impl MmCtx { /// Returns whether node is configured to use [Upgraded Trading Protocol](https://github.com/KomodoPlatform/komodo-defi-framework/issues/1895) pub fn use_trading_proto_v2(&self) -> bool { self.conf["use_trading_proto_v2"].as_bool().unwrap_or_default() } - /// Returns the cloneable `MmFutSpawner`. - pub fn spawner(&self) -> MmFutSpawner { MmFutSpawner::new(&self.abortable_system) } + /// Returns the cloneable `WeakSpawner`. + pub fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } /// True if the MarketMaker instance needs to stop. pub fn is_stopping(&self) -> bool { self.stop.copy_or(false) } @@ -640,44 +640,6 @@ impl MmArc { } } -/// The futures spawner pinned to the `MmCtx` context. -/// It's used to spawn futures that can be aborted immediately or after a timeout -/// on the [`MmArc::stop`] function call. -/// -/// # Note -/// -/// `MmFutSpawner` doesn't prevent the spawned futures from being aborted. -#[derive(Clone)] -pub struct MmFutSpawner { - inner: WeakSpawner, -} - -impl MmFutSpawner { - pub fn new(system: &AbortableQueue) -> MmFutSpawner { - MmFutSpawner { - inner: system.weak_spawner(), - } - } -} - -impl SpawnFuture for MmFutSpawner { - fn spawn(&self, f: F) - where - F: Future + Send + 'static, - { - self.inner.spawn(f) - } -} - -impl SpawnAbortable for MmFutSpawner { - fn spawn_with_settings(&self, fut: F, settings: AbortSettings) - where - F: Future + Send + 'static, - { - self.inner.spawn_with_settings(fut, settings) - } -} - /// Helps getting a crate context from a corresponding `MmCtx` field. /// /// * `ctx_field` - A dedicated crate context field in `MmCtx`, such as the `MmCtx::portfolio_ctx`. diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 72d0e063d5..c2b20ecbc1 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -430,17 +430,17 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { let network_steamer = NetworkEvent::try_new(config, ctx.clone()) .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))?; ctx.event_stream_manager - // FIXME: weak doesnt work, get rid of mmfutspawner & coinfutspawner and replace them with plain week spawner. - .add(network_steamer, ctx.spawner().weak()) + .add(network_steamer, ctx.spawner()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e}")))?; } if let Some(config) = ctx.event_stream_configuration.get_event("HEARTBEAT") { - let heartbeat_streamer = HeartbeatEvent::try_new(config, ctx.clone()) - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")))?; + let heartbeat_streamer = HeartbeatEvent::try_new(config, ctx.clone()).map_to_mm(|e| { + MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")) + })?; ctx.event_stream_manager - .add(heartbeat_streamer, ctx.spawner().weak()) + .add(heartbeat_streamer, ctx.spawner()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e}")))?; } From 8af7abc301aa54612a84f251353017e50cba6814 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 14:02:39 +0200 Subject: [PATCH 024/128] fix compilation issues --- mm2src/coins/eth.rs | 1 - mm2src/coins/eth/eth_balance_events.rs | 24 +++++++------------ mm2src/coins/lp_coins.rs | 4 +--- .../tendermint/tendermint_balance_events.rs | 13 +++++----- mm2src/coins/utxo/utxo_balance_events.rs | 19 +++++++-------- .../utxo/utxo_builder/utxo_arc_builder.rs | 1 - mm2src/coins/z_coin.rs | 1 - mm2src/coins/z_coin/z_balance_streaming.rs | 15 ++++++------ .../src/tendermint_with_assets_activation.rs | 1 - .../src/utxo_activation/common_impl.rs | 2 +- mm2src/mm2_core/src/mm_ctx.rs | 3 +-- mm2src/mm2_main/src/heartbeat_event.rs | 2 +- mm2src/mm2_main/src/lp_native_dex.rs | 1 - 13 files changed, 35 insertions(+), 52 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 67065d57fb..dc93eb3f2b 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -79,7 +79,6 @@ use http::Uri; use instant::Instant; use keys::Public as HtlcPubKey; use mm2_core::mm_ctx::{MmArc, MmWeak}; -use mm2_event_stream::EventStreamer; use mm2_net::transport::{GuiAuthValidation, GuiAuthValidationGenerator}; use mm2_number::bigdecimal_custom::CheckedDivision; use mm2_number::{BigDecimal, BigUint, MmNumber}; diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 79658d4e6c..aea54e6812 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -1,12 +1,11 @@ use async_trait::async_trait; -use common::{executor::{AbortSettings, SpawnAbortable, Timer}, - log, Future01CompatExt}; +use common::{executor::Timer, log, Future01CompatExt}; use ethereum_types::Address; use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Event, EventName, EventStreamer, NoDataIn}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -15,7 +14,7 @@ use std::collections::{HashMap, HashSet}; use super::EthCoin; use crate::streaming_events_config::BalanceEventConfig; use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, - BalanceError, CoinWithDerivationMethod, MmCoin}; + BalanceError, CoinWithDerivationMethod}; #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -152,11 +151,10 @@ impl EventStreamer for EthBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker) } - async fn handle(self, tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - let streamer_id = self.streamer_id(); - async fn start_polling(coin: EthCoin, ctx: MmArc, interval: f64) { + async fn start_polling(streamer_id: String, coin: EthCoin, ctx: MmArc, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, // our goal is to broadcast changed balances every x seconds. @@ -224,11 +222,7 @@ impl EventStreamer for EthBalanceEventStreamer { if !balance_updates.is_empty() { ctx.stream_channel_controller - .broadcast(Event::new( - EthBalanceEventStreamer::event_name().to_string(), - json!(balance_updates), - None, - )) + .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) .await; } @@ -240,13 +234,13 @@ impl EventStreamer for EthBalanceEventStreamer { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; - tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - start_polling(self.coin, ctx, self.interval).await + start_polling(self.streamer_id(), self.coin, ctx, self.interval).await } } diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 356503b6f2..98e53a7b14 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -45,8 +45,7 @@ use async_trait::async_trait; use base58::FromBase58Error; use bip32::ExtendedPrivateKey; use common::custom_futures::timeout::TimeoutError; -use common::executor::{abortable_queue::{AbortableQueue, WeakSpawner}, - AbortSettings, AbortedError, SpawnAbortable, SpawnFuture}; +use common::executor::{abortable_queue::WeakSpawner, AbortedError, SpawnFuture}; use common::log::{warn, LogOnError}; use common::{calc_total_pages, now_sec, ten, HttpStatusCode}; use crypto::{derive_secp256k1_secret, Bip32Error, Bip44Chain, CryptoCtx, CryptoCtxError, DerivationPath, @@ -75,7 +74,6 @@ use serde_json::{self as json, Value as Json}; use std::cmp::Ordering; use std::collections::hash_map::{HashMap, RawEntryMut}; use std::collections::HashSet; -use std::future::Future as Future03; use std::num::{NonZeroUsize, TryFromIntError}; use std::ops::{Add, AddAssign, Deref}; use std::str::FromStr; diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 55e5bdf176..05ea0cb8b4 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -1,6 +1,5 @@ use async_trait::async_trait; -use common::{executor::{AbortSettings, SpawnAbortable}, - http_uri_to_ws_address, log}; +use common::{http_uri_to_ws_address, log}; use futures::channel::oneshot; use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; @@ -13,7 +12,7 @@ use std::collections::{HashMap, HashSet}; use super::TendermintCoin; use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; -use crate::{tendermint::TendermintCommons, utxo::utxo_common::big_decimal_from_sat_unsigned, MarketCoinOps, MmCoin}; +use crate::{tendermint::TendermintCommons, utxo::utxo_common::big_decimal_from_sat_unsigned, MarketCoinOps}; pub struct TendermintBalanceEventStreamer { /// Whether the event is enabled for this coin. @@ -39,9 +38,9 @@ impl EventStreamer for TendermintBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } - async fn handle(self, tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - let streamer_id = self.streaemr_id(); + let streamer_id = self.streamer_id(); let coin = self.coin; fn generate_subscription_query(query_filter: String) -> String { @@ -62,7 +61,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; @@ -76,7 +75,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { let spender_q = generate_subscription_query(format!("coin_spent.spender = '{}'", account_id)); let spender_q = tokio_tungstenite_wasm::Message::Text(spender_q); - tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); loop { let node_uri = match coin.rpc_client().await { diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index d02ddf0662..98fa3efe78 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -1,11 +1,10 @@ use async_trait::async_trait; -use common::{executor::{AbortSettings, SpawnAbortable, Timer}, - log, Future01CompatExt}; +use common::{executor::Timer, log, Future01CompatExt}; use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventName, EventStreamer, Filter}; +use mm2_event_stream::{Event, EventStreamer, Filter, StreamHandlerInput}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -16,7 +15,7 @@ use crate::{utxo::{output_script, rpc_clients::electrum_script_hash, utxo_common::{address_balance, address_to_scripthash}, ScripthashNotification, UtxoCoinFields}, - CoinWithDerivationMethod, MarketCoinOps, MmCoin}; + CoinWithDerivationMethod, MarketCoinOps}; macro_rules! try_or_continue { ($exp:expr) => { @@ -48,7 +47,7 @@ impl UtxoBalanceEventStreamer { enabled, // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. // The UtxoArc might belong to a different coin type though. - coin: UtxoStandardCoin::from(utxo_arc).spawner(), + coin: UtxoStandardCoin::from(utxo_arc), }) } } @@ -61,7 +60,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { async fn handle( self, - tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; @@ -109,7 +108,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { if coin.as_ref().rpc_client.is_native() { log::warn!("Native RPC client is not supported for {streamer_id} event. Skipping event initialization."); // We won't consider this an error but just an unsupported scenario and continue running. - tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); panic!("Native RPC client is not supported for UtxoBalanceEventStreamer."); } @@ -117,7 +116,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { Some(ctx) => ctx, None => { let msg = "MM context must have been initialized already."; - tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); panic!("{}", msg); }, }; @@ -126,12 +125,12 @@ impl EventStreamer for UtxoBalanceEventStreamer { Some(t) => t, None => { let e = "Scripthash notification receiver can not be empty."; - tx.send(Err(e.to_string())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Err(e.to_string())).expect(RECEIVER_DROPPED_MSG); panic!("{}", e); }, }; - tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); let mut scripthash_to_address_map = BTreeMap::default(); while let Some(message) = scripthash_notification_handler.lock().await.next().await { diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index f4af90fbd6..340d726bf4 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -14,7 +14,6 @@ use common::log::{debug, error, info, warn}; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; #[cfg(test)] use mocktopus::macros::*; use rand::Rng; use script::Builder; diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 369a9a9319..8624f47de0 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -60,7 +60,6 @@ use keys::hash::H256; use keys::{KeyPair, Message, Public}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; use primitives::bytes::Bytes; diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 9237cfd7a8..86ee0f75d3 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -1,17 +1,16 @@ use crate::common::Future01CompatExt; use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::z_coin::ZCoin; -use crate::{MarketCoinOps, MmCoin}; +use crate::MarketCoinOps; use async_trait::async_trait; -use common::executor::{AbortSettings, SpawnAbortable}; -use common::log::{error, info}; +use common::log::error; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, StreamerHandleInput}; +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput}; use serde_json::Value as Json; use std::sync::Arc; @@ -42,7 +41,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } - async fn handle(self, tx: oneshot::Sender>, data_rx: impl StreamerHandleInput<()>) { + async fn handle(self, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput<()>) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); let coin = self.coin; @@ -61,16 +60,16 @@ impl EventStreamer for ZCoinBalanceEventStreamer { let ctx = send_status_on_err!( MmArc::from_weak(&coin.as_ref().ctx), - tx, + ready_tx, "MM context must have been initialized already." ); let z_balance_change_handler = send_status_on_err!( coin.z_fields.z_balance_event_handler.as_ref(), - tx, + ready_tx, "Z balance change receiver can not be empty." ); - tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); // Locks the balance change handler, iterates through received events, and updates balance changes accordingly. let mut bal = z_balance_change_handler.lock().await; diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 1b1605b006..d3363cab01 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -22,7 +22,6 @@ use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use mm2_event_stream::EventStreamConfiguration; -use mm2_event_stream::EventStreamer; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde::{Deserialize, Deserializer, Serialize}; diff --git a/mm2src/coins_activation/src/utxo_activation/common_impl.rs b/mm2src/coins_activation/src/utxo_activation/common_impl.rs index e22428264a..d5c9be6694 100644 --- a/mm2src/coins_activation/src/utxo_activation/common_impl.rs +++ b/mm2src/coins_activation/src/utxo_activation/common_impl.rs @@ -8,7 +8,7 @@ use coins::hd_wallet::RpcTaskXPubExtractor; use coins::my_tx_history_v2::TxHistoryStorage; use coins::utxo::utxo_tx_history_v2::{utxo_history_loop, UtxoTxHistoryOps}; use coins::utxo::{UtxoActivationParams, UtxoCoinFields}; -use coins::{CoinBalance, MarketCoinOps, PrivKeyActivationPolicy, PrivKeyBuildPolicy, WeakSpawner}; +use coins::{CoinBalance, MarketCoinOps, PrivKeyActivationPolicy, PrivKeyBuildPolicy}; use common::executor::{AbortSettings, SpawnAbortable}; use crypto::hw_rpc_task::HwConnectStatuses; use crypto::{CryptoCtxError, HwRpcError}; diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index c81d3c1592..5bf9ea62a5 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -1,7 +1,7 @@ #[cfg(feature = "track-ctx-pointer")] use common::executor::Timer; use common::executor::{abortable_queue::{AbortableQueue, WeakSpawner}, - graceful_shutdown, AbortSettings, AbortableSystem, SpawnAbortable, SpawnFuture}; + graceful_shutdown, AbortableSystem}; use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; @@ -16,7 +16,6 @@ use std::any::Any; use std::collections::hash_map::{Entry, HashMap}; use std::collections::HashSet; use std::fmt; -use std::future::Future; use std::ops::Deref; use std::sync::{Arc, Mutex}; diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 02d67de26f..b9a6ab6a2a 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -37,7 +37,7 @@ impl EventStreamer for HeartbeatEvent { fn streamer_id(&self) -> String { "HEARTBEAT".to_string() } async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { - tx.send(Ok(())).unwrap(); + ready_tx.send(Ok(())).unwrap(); loop { self.ctx diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index c2b20ecbc1..86a2e0b261 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -28,7 +28,6 @@ use enum_derives::EnumFromTrait; use mm2_core::mm_ctx::{MmArc, MmCtx}; use mm2_err_handle::common_errors::InternalError; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_libp2p::behaviours::atomicdex::{GossipsubConfig, DEPRECATED_NETID_LIST}; use mm2_libp2p::{spawn_gossipsub, AdexBehaviourError, NodeType, RelayAddress, RelayAddressError, SeedNodeInfo, SwarmRuntime, WssCerts}; From 3ade9a2c3e6678610e49f8daff784586ac907559 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 15:51:52 +0200 Subject: [PATCH 025/128] StreamHandlerInput must be Unpin to call next() Also use the `data_rx` for utxo balance events. A cleanup for the scripthash notification channels/logic will follow --- mm2src/coins/utxo/utxo_balance_events.rs | 13 ++----------- mm2src/mm2_event_stream/src/streamer.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 98fa3efe78..fe46426de9 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -61,7 +61,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { async fn handle( self, ready_tx: oneshot::Sender>, - data_rx: impl StreamHandlerInput, + mut data_rx: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); @@ -121,19 +121,10 @@ impl EventStreamer for UtxoBalanceEventStreamer { }, }; - let scripthash_notification_handler = match coin.as_ref().scripthash_notification_handler.as_ref() { - Some(t) => t, - None => { - let e = "Scripthash notification receiver can not be empty."; - ready_tx.send(Err(e.to_string())).expect(RECEIVER_DROPPED_MSG); - panic!("{}", e); - }, - }; - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); let mut scripthash_to_address_map = BTreeMap::default(); - while let Some(message) = scripthash_notification_handler.lock().await.next().await { + while let Some(message) = data_rx.next().await { let notified_scripthash = match message { ScripthashNotification::Triggered(t) => t, ScripthashNotification::SubscribeToAddresses(addresses) => { diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index ff68033a3b..09812c0b58 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -10,10 +10,10 @@ use futures::{select, FutureExt, Stream, StreamExt}; /// A marker to indicate that the event streamer doesn't take any input data. pub struct NoDataIn; -/// A mixture trait combining `Stream` and `Send` together (to avoid confusing annotation). -pub trait StreamHandlerInput: Stream + Send {} -/// Implement the trait for all types `T` that implement `Stream + Send` for all `D`. -impl StreamHandlerInput for T where T: Stream + Send {} +/// A mixture trait combining `Stream`, `Send` & `Unpin` together (to avoid confusing annotation). +pub trait StreamHandlerInput: Stream + Send + Unpin {} +/// Implement the trait for all types `T` that implement `Stream + Send + Unpin` for any `D`. +impl StreamHandlerInput for T where T: Stream + Send + Unpin {} #[async_trait] pub trait EventStreamer @@ -63,14 +63,14 @@ where let streamer_id = streamer_id.clone(); move |any_input_data| { let streamer_id = streamer_id.clone(); - async move { + Box::pin(async move { if let Ok(input_data) = any_input_data.downcast() { Some(*input_data) } else { error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); None } - } + }) } }); From 8972822af21587b02e2285857f8f4d4d6ba280cc Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 18:04:48 +0200 Subject: [PATCH 026/128] PoC: utxo balance streaming using the new streaming manager interface this is very messy though (because of the current state of electurm code) and will be reverted, just recording this in the history for now --- mm2src/coins/utxo.rs | 7 +-- mm2src/coins/utxo/rpc_clients.rs | 57 ++++++++++++------- .../utxo/utxo_builder/utxo_coin_builder.rs | 4 +- mm2src/coins/utxo/utxo_common_tests.rs | 1 - mm2src/coins/utxo/utxo_tests.rs | 3 +- mm2src/mm2_core/src/mm_ctx.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 1 + mm2src/mm2_event_stream/src/manager.rs | 3 +- 8 files changed, 43 insertions(+), 35 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index da5e640e23..2f1119959d 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -147,8 +147,7 @@ pub enum ScripthashNotification { RefreshSubscriptions, } -pub type ScripthashNotificationSender = Option>; -type ScripthashNotificationHandler = Option>>>; +type ScripthashNotificationSender = Option>; #[cfg(windows)] #[cfg(not(target_arch = "wasm32"))] @@ -619,10 +618,6 @@ pub struct UtxoCoinFields { /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, pub(crate) ctx: MmWeak, - /// This is used for balance event streaming implementation for UTXOs. - /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, - /// it will be used for receiving scripthash notifications to re-fetch balances. - scripthash_notification_handler: ScripthashNotificationHandler, } #[derive(Debug, Display)] diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index f5150e4a9a..0de9dfa9d7 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -33,6 +33,7 @@ use itertools::Itertools; use keys::hash::H256; use keys::Address; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_number::{BigDecimal, BigInt, MmNumber}; use mm2_rpc::data::legacy::ElectrumProtocol; #[cfg(test)] use mocktopus::macros::*; @@ -1479,7 +1480,8 @@ fn server_name_from_domain(dns_name: &str) -> Result { pub fn spawn_electrum( req: &ElectrumRpcRequest, event_handlers: Vec, - scripthash_notification_sender: &ScripthashNotificationSender, + streaming_manager: StreamingManager, + ticker: String, abortable_system: AbortableQueue, ) -> Result { let config = match req.protocol { @@ -1506,7 +1508,8 @@ pub fn spawn_electrum( req.url.clone(), config, event_handlers, - scripthash_notification_sender, + streaming_manager, + ticker, abortable_system, )) } @@ -1660,6 +1663,8 @@ pub struct ElectrumClientImpl { /// Please also note that this abortable system is a subsystem of [`UtxoCoinFields::abortable_system`]. abortable_system: AbortableQueue, negotiate_version: bool, + /// This is a copy of the streaming manager so to be able to communicate with utxo balance streamer. + streaming_manager: StreamingManager, /// This is used for balance event streaming implementation for UTXOs. /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, /// it will be used for sending scripthash messages to trigger re-connections, re-fetching the balances, etc. @@ -1756,7 +1761,8 @@ impl ElectrumClientImpl { let connection = try_s!(spawn_electrum( req, self.event_handlers.clone(), - &self.scripthash_notification_sender, + self.streaming_manager.clone(), + self.coin_ticker.clone(), subsystem, )); self.connections.lock().await.push(connection); @@ -2515,7 +2521,7 @@ impl ElectrumClientImpl { block_headers_storage: BlockHeaderStorage, abortable_system: AbortableQueue, negotiate_version: bool, - scripthash_notification_sender: ScripthashNotificationSender, + streaming_manager: StreamingManager, ) -> ElectrumClientImpl { let protocol_version = OrdRange::new(1.2, 1.4).unwrap(); ElectrumClientImpl { @@ -2529,7 +2535,8 @@ impl ElectrumClientImpl { block_headers_storage, abortable_system, negotiate_version, - scripthash_notification_sender, + streaming_manager, + scripthash_notification_sender: None, } } @@ -2540,7 +2547,7 @@ impl ElectrumClientImpl { protocol_version: OrdRange, block_headers_storage: BlockHeaderStorage, abortable_system: AbortableQueue, - scripthash_notification_sender: ScripthashNotificationSender, + streaming_manager: StreamingManager, ) -> ElectrumClientImpl { ElectrumClientImpl { protocol_version, @@ -2550,7 +2557,7 @@ impl ElectrumClientImpl { block_headers_storage, abortable_system, false, - scripthash_notification_sender, + streaming_manager, ) } } @@ -2564,7 +2571,8 @@ fn rx_to_stream(rx: mpsc::Receiver>) -> impl Stream, Erro async fn electrum_process_json( raw_json: Json, arc: &JsonRpcPendingRequestsShared, - scripthash_notification_sender: &ScripthashNotificationSender, + streaming_manager: &StreamingManager, + ticker: &str, ) { // detect if we got standard JSONRPC response or subscription response as JSONRPC request #[derive(Deserialize)] @@ -2605,14 +2613,14 @@ async fn electrum_process_json( }, }; - if let Some(sender) = scripthash_notification_sender { - debug!("Sending scripthash message"); - if let Err(e) = sender.unbounded_send(ScripthashNotification::Triggered(scripthash.to_string())) - { - error!("Failed sending scripthash message. {e}"); - return; - }; + if let Err(e) = streaming_manager.send( + &format!("BALANCE:{ticker}"), + ScripthashNotification::Triggered(scripthash.to_string()), + ) { + error!("Failed sending scripthash message. {e:?}"); + return; }; + BLOCKCHAIN_SCRIPTHASH_SUB_ID }, _ => { @@ -2640,7 +2648,8 @@ async fn electrum_process_json( async fn electrum_process_chunk( chunk: &[u8], arc: &JsonRpcPendingRequestsShared, - scripthash_notification_sender: ScripthashNotificationSender, + streaming_manager: &StreamingManager, + ticker: &str, ) { // we should split the received chunk because we can get several responses in 1 chunk. let split = chunk.split(|item| *item == b'\n'); @@ -2654,7 +2663,7 @@ async fn electrum_process_chunk( return; }, }; - electrum_process_json(raw_json, arc, &scripthash_notification_sender).await + electrum_process_json(raw_json, arc, streaming_manager, ticker).await } } } @@ -2782,7 +2791,8 @@ async fn connect_loop( responses: JsonRpcPendingRequestsShared, connection_tx: Arc>>>>, event_handlers: Vec, - scripthash_notification_sender: ScripthashNotificationSender, + streaming_manager: StreamingManager, + ticker: String, _spawner: Spawner, ) -> Result<(), ()> { let delay = Arc::new(AtomicU64::new(0)); @@ -2839,7 +2849,8 @@ async fn connect_loop( let delay = delay.clone(); let addr = addr.clone(); let responses = responses.clone(); - let scripthash_notification_sender = scripthash_notification_sender.clone(); + let streaming_manager = streaming_manager.clone(); + let ticker = ticker.clone(); let event_handlers = event_handlers.clone(); async move { let mut buffer = String::with_capacity(1024); @@ -2863,7 +2874,7 @@ async fn connect_loop( event_handlers.on_incoming_response(buffer.as_bytes()); last_chunk.store(now_ms(), AtomicOrdering::Relaxed); - electrum_process_chunk(buffer.as_bytes(), &responses, scripthash_notification_sender.clone()).await; + electrum_process_chunk(buffer.as_bytes(), &responses, &streaming_manager, &ticker).await; buffer.clear(); } } @@ -3015,7 +3026,8 @@ fn electrum_connect( addr: String, config: ElectrumConfig, event_handlers: Vec, - scripthash_notification_sender: &ScripthashNotificationSender, + streaming_manager: StreamingManager, + ticker: String, abortable_system: AbortableQueue, ) -> ElectrumConnection { let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); @@ -3028,7 +3040,8 @@ fn electrum_connect( responses.clone(), tx.clone(), event_handlers, - scripthash_notification_sender.clone(), + streaming_manager, + ticker, spawner.clone(), ) .then(|_| futures::future::ready(())); diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 2cbb57b35c..1c18293a50 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -304,7 +304,6 @@ where block_headers_status_notifier, block_headers_status_watcher, abortable_system, - scripthash_notification_handler, ctx: builder.ctx().weak(), }; @@ -390,7 +389,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { block_headers_status_notifier, block_headers_status_watcher, abortable_system, - scripthash_notification_handler, ctx: self.ctx().weak(), }; Ok(coin) @@ -596,7 +594,7 @@ pub trait UtxoCoinBuilderCommonOps { block_headers_storage, abortable_system, args.negotiate_version, - scripthash_notification_sender, + ctx.event_stream_manager.clone(), ); for server in servers.iter() { match client.add_server(server).await { diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index 03fea53699..046045ede4 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -148,7 +148,6 @@ pub(super) fn utxo_coin_fields_for_test( block_headers_status_notifier: None, block_headers_status_watcher: None, abortable_system: AbortableQueue::default(), - scripthash_notification_handler: None, ctx: Default::default(), } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 868f558f1f..65bc1575e6 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -43,6 +43,7 @@ use futures::future::join_all; use futures::TryFutureExt; use keys::prefixes::*; use mm2_core::mm_ctx::MmCtxBuilder; +use mm2_event_stream::StreamingManager; use mm2_number::bigdecimal::{BigDecimal, Signed}; use mm2_test_helpers::electrums::doc_electrums; use mm2_test_helpers::for_tests::{electrum_servers_rpc, mm_ctx_with_custom_db, DOC_ELECTRUM_ADDRS, @@ -477,7 +478,7 @@ fn test_wait_for_payment_spend_timeout_electrum() { block_headers_storage, abortable_system, true, - None, + StreamingManager::default(), ); let client = UtxoRpcClientEnum::Electrum(ElectrumClient(Arc::new(client))); let coin = utxo_coin_for_test(client, None, false); diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 5bf9ea62a5..6294a5e224 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{manager::StreamingManager, Controller, Event, EventStreamConfiguration}; +use mm2_event_stream::{Controller, Event, EventStreamConfiguration, StreamingManager}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index ddec11d74e..45c9baf8f6 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -8,4 +8,5 @@ pub mod streamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, EventName, Filter}; +pub use manager::{StreamingManager, StreamingSendError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 338fdec74a..95e1702298 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -9,6 +9,7 @@ use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; /// The errors that could originate from the streaming manager. +#[derive(Debug)] pub enum StreamingSendError { /// There is no streamer with the given ID. StreamerNotFound, @@ -18,7 +19,7 @@ pub enum StreamingSendError { NoDataIn, } -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: Arc, Option>>)>>>, From bf54981d87e8bb1d6c0f1821742e570e3b3422bc Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 18:04:57 +0200 Subject: [PATCH 027/128] Revert "PoC: utxo balance streaming using the new streaming manager interface" This reverts commit 8972822af21587b02e2285857f8f4d4d6ba280cc. --- mm2src/coins/utxo.rs | 7 ++- mm2src/coins/utxo/rpc_clients.rs | 57 +++++++------------ .../utxo/utxo_builder/utxo_coin_builder.rs | 4 +- mm2src/coins/utxo/utxo_common_tests.rs | 1 + mm2src/coins/utxo/utxo_tests.rs | 3 +- mm2src/mm2_core/src/mm_ctx.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 1 - mm2src/mm2_event_stream/src/manager.rs | 3 +- 8 files changed, 35 insertions(+), 43 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 2f1119959d..da5e640e23 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -147,7 +147,8 @@ pub enum ScripthashNotification { RefreshSubscriptions, } -type ScripthashNotificationSender = Option>; +pub type ScripthashNotificationSender = Option>; +type ScripthashNotificationHandler = Option>>>; #[cfg(windows)] #[cfg(not(target_arch = "wasm32"))] @@ -618,6 +619,10 @@ pub struct UtxoCoinFields { /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, pub(crate) ctx: MmWeak, + /// This is used for balance event streaming implementation for UTXOs. + /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, + /// it will be used for receiving scripthash notifications to re-fetch balances. + scripthash_notification_handler: ScripthashNotificationHandler, } #[derive(Debug, Display)] diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 0de9dfa9d7..f5150e4a9a 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -33,7 +33,6 @@ use itertools::Itertools; use keys::hash::H256; use keys::Address; use mm2_err_handle::prelude::*; -use mm2_event_stream::StreamingManager; use mm2_number::{BigDecimal, BigInt, MmNumber}; use mm2_rpc::data::legacy::ElectrumProtocol; #[cfg(test)] use mocktopus::macros::*; @@ -1480,8 +1479,7 @@ fn server_name_from_domain(dns_name: &str) -> Result { pub fn spawn_electrum( req: &ElectrumRpcRequest, event_handlers: Vec, - streaming_manager: StreamingManager, - ticker: String, + scripthash_notification_sender: &ScripthashNotificationSender, abortable_system: AbortableQueue, ) -> Result { let config = match req.protocol { @@ -1508,8 +1506,7 @@ pub fn spawn_electrum( req.url.clone(), config, event_handlers, - streaming_manager, - ticker, + scripthash_notification_sender, abortable_system, )) } @@ -1663,8 +1660,6 @@ pub struct ElectrumClientImpl { /// Please also note that this abortable system is a subsystem of [`UtxoCoinFields::abortable_system`]. abortable_system: AbortableQueue, negotiate_version: bool, - /// This is a copy of the streaming manager so to be able to communicate with utxo balance streamer. - streaming_manager: StreamingManager, /// This is used for balance event streaming implementation for UTXOs. /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, /// it will be used for sending scripthash messages to trigger re-connections, re-fetching the balances, etc. @@ -1761,8 +1756,7 @@ impl ElectrumClientImpl { let connection = try_s!(spawn_electrum( req, self.event_handlers.clone(), - self.streaming_manager.clone(), - self.coin_ticker.clone(), + &self.scripthash_notification_sender, subsystem, )); self.connections.lock().await.push(connection); @@ -2521,7 +2515,7 @@ impl ElectrumClientImpl { block_headers_storage: BlockHeaderStorage, abortable_system: AbortableQueue, negotiate_version: bool, - streaming_manager: StreamingManager, + scripthash_notification_sender: ScripthashNotificationSender, ) -> ElectrumClientImpl { let protocol_version = OrdRange::new(1.2, 1.4).unwrap(); ElectrumClientImpl { @@ -2535,8 +2529,7 @@ impl ElectrumClientImpl { block_headers_storage, abortable_system, negotiate_version, - streaming_manager, - scripthash_notification_sender: None, + scripthash_notification_sender, } } @@ -2547,7 +2540,7 @@ impl ElectrumClientImpl { protocol_version: OrdRange, block_headers_storage: BlockHeaderStorage, abortable_system: AbortableQueue, - streaming_manager: StreamingManager, + scripthash_notification_sender: ScripthashNotificationSender, ) -> ElectrumClientImpl { ElectrumClientImpl { protocol_version, @@ -2557,7 +2550,7 @@ impl ElectrumClientImpl { block_headers_storage, abortable_system, false, - streaming_manager, + scripthash_notification_sender, ) } } @@ -2571,8 +2564,7 @@ fn rx_to_stream(rx: mpsc::Receiver>) -> impl Stream, Erro async fn electrum_process_json( raw_json: Json, arc: &JsonRpcPendingRequestsShared, - streaming_manager: &StreamingManager, - ticker: &str, + scripthash_notification_sender: &ScripthashNotificationSender, ) { // detect if we got standard JSONRPC response or subscription response as JSONRPC request #[derive(Deserialize)] @@ -2613,14 +2605,14 @@ async fn electrum_process_json( }, }; - if let Err(e) = streaming_manager.send( - &format!("BALANCE:{ticker}"), - ScripthashNotification::Triggered(scripthash.to_string()), - ) { - error!("Failed sending scripthash message. {e:?}"); - return; + if let Some(sender) = scripthash_notification_sender { + debug!("Sending scripthash message"); + if let Err(e) = sender.unbounded_send(ScripthashNotification::Triggered(scripthash.to_string())) + { + error!("Failed sending scripthash message. {e}"); + return; + }; }; - BLOCKCHAIN_SCRIPTHASH_SUB_ID }, _ => { @@ -2648,8 +2640,7 @@ async fn electrum_process_json( async fn electrum_process_chunk( chunk: &[u8], arc: &JsonRpcPendingRequestsShared, - streaming_manager: &StreamingManager, - ticker: &str, + scripthash_notification_sender: ScripthashNotificationSender, ) { // we should split the received chunk because we can get several responses in 1 chunk. let split = chunk.split(|item| *item == b'\n'); @@ -2663,7 +2654,7 @@ async fn electrum_process_chunk( return; }, }; - electrum_process_json(raw_json, arc, streaming_manager, ticker).await + electrum_process_json(raw_json, arc, &scripthash_notification_sender).await } } } @@ -2791,8 +2782,7 @@ async fn connect_loop( responses: JsonRpcPendingRequestsShared, connection_tx: Arc>>>>, event_handlers: Vec, - streaming_manager: StreamingManager, - ticker: String, + scripthash_notification_sender: ScripthashNotificationSender, _spawner: Spawner, ) -> Result<(), ()> { let delay = Arc::new(AtomicU64::new(0)); @@ -2849,8 +2839,7 @@ async fn connect_loop( let delay = delay.clone(); let addr = addr.clone(); let responses = responses.clone(); - let streaming_manager = streaming_manager.clone(); - let ticker = ticker.clone(); + let scripthash_notification_sender = scripthash_notification_sender.clone(); let event_handlers = event_handlers.clone(); async move { let mut buffer = String::with_capacity(1024); @@ -2874,7 +2863,7 @@ async fn connect_loop( event_handlers.on_incoming_response(buffer.as_bytes()); last_chunk.store(now_ms(), AtomicOrdering::Relaxed); - electrum_process_chunk(buffer.as_bytes(), &responses, &streaming_manager, &ticker).await; + electrum_process_chunk(buffer.as_bytes(), &responses, scripthash_notification_sender.clone()).await; buffer.clear(); } } @@ -3026,8 +3015,7 @@ fn electrum_connect( addr: String, config: ElectrumConfig, event_handlers: Vec, - streaming_manager: StreamingManager, - ticker: String, + scripthash_notification_sender: &ScripthashNotificationSender, abortable_system: AbortableQueue, ) -> ElectrumConnection { let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); @@ -3040,8 +3028,7 @@ fn electrum_connect( responses.clone(), tx.clone(), event_handlers, - streaming_manager, - ticker, + scripthash_notification_sender.clone(), spawner.clone(), ) .then(|_| futures::future::ready(())); diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 1c18293a50..2cbb57b35c 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -304,6 +304,7 @@ where block_headers_status_notifier, block_headers_status_watcher, abortable_system, + scripthash_notification_handler, ctx: builder.ctx().weak(), }; @@ -389,6 +390,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { block_headers_status_notifier, block_headers_status_watcher, abortable_system, + scripthash_notification_handler, ctx: self.ctx().weak(), }; Ok(coin) @@ -594,7 +596,7 @@ pub trait UtxoCoinBuilderCommonOps { block_headers_storage, abortable_system, args.negotiate_version, - ctx.event_stream_manager.clone(), + scripthash_notification_sender, ); for server in servers.iter() { match client.add_server(server).await { diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index 046045ede4..03fea53699 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -148,6 +148,7 @@ pub(super) fn utxo_coin_fields_for_test( block_headers_status_notifier: None, block_headers_status_watcher: None, abortable_system: AbortableQueue::default(), + scripthash_notification_handler: None, ctx: Default::default(), } } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 65bc1575e6..868f558f1f 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -43,7 +43,6 @@ use futures::future::join_all; use futures::TryFutureExt; use keys::prefixes::*; use mm2_core::mm_ctx::MmCtxBuilder; -use mm2_event_stream::StreamingManager; use mm2_number::bigdecimal::{BigDecimal, Signed}; use mm2_test_helpers::electrums::doc_electrums; use mm2_test_helpers::for_tests::{electrum_servers_rpc, mm_ctx_with_custom_db, DOC_ELECTRUM_ADDRS, @@ -478,7 +477,7 @@ fn test_wait_for_payment_spend_timeout_electrum() { block_headers_storage, abortable_system, true, - StreamingManager::default(), + None, ); let client = UtxoRpcClientEnum::Electrum(ElectrumClient(Arc::new(client))); let coin = utxo_coin_for_test(client, None, false); diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 6294a5e224..5bf9ea62a5 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{Controller, Event, EventStreamConfiguration, StreamingManager}; +use mm2_event_stream::{manager::StreamingManager, Controller, Event, EventStreamConfiguration}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 45c9baf8f6..ddec11d74e 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -8,5 +8,4 @@ pub mod streamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, EventName, Filter}; -pub use manager::{StreamingManager, StreamingSendError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 95e1702298..338fdec74a 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -9,7 +9,6 @@ use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; /// The errors that could originate from the streaming manager. -#[derive(Debug)] pub enum StreamingSendError { /// There is no streamer with the given ID. StreamerNotFound, @@ -19,7 +18,7 @@ pub enum StreamingSendError { NoDataIn, } -#[derive(Clone, Default, Debug)] +#[derive(Clone, Default)] pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: Arc, Option>>)>>>, From cf503896165917fdfd02cfc2b9f5b29c4a381995 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 16 Jul 2024 18:54:07 +0200 Subject: [PATCH 028/128] hide the Controller inside the StreamingManager made more sense to have it inside the manager. also now the manager provides the controller as a parameter in the handle method so different streamers don't have to keep a copy of ctx just to broadcast on the controller. --- mm2src/coins/eth/eth_balance_events.rs | 29 ++++++++++++------- .../tendermint/tendermint_balance_events.rs | 15 ++++++---- mm2src/coins/utxo/utxo_balance_events.rs | 11 +++---- mm2src/coins/z_coin/z_balance_streaming.rs | 16 +++++----- mm2src/mm2_core/src/data_asker.rs | 2 +- mm2src/mm2_core/src/mm_ctx.rs | 3 -- mm2src/mm2_event_stream/src/manager.rs | 17 +++++++++-- mm2src/mm2_event_stream/src/streamer.rs | 5 +++- mm2src/mm2_main/src/heartbeat_event.rs | 16 +++++----- mm2src/mm2_main/src/lp_native_dex.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 12 +++++--- mm2src/mm2_net/src/sse_handler.rs | 2 +- 12 files changed, 81 insertions(+), 49 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index aea54e6812..4ad1f5d72e 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -5,7 +5,7 @@ use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -151,10 +151,21 @@ impl EventStreamer for EthBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker) } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle( + self, + broadcaster: Controller, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - async fn start_polling(streamer_id: String, coin: EthCoin, ctx: MmArc, interval: f64) { + async fn start_polling( + streamer_id: String, + broadcaster: Controller, + coin: EthCoin, + ctx: MmArc, + interval: f64, + ) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, // our goal is to broadcast changed balances every x seconds. @@ -178,9 +189,7 @@ impl EventStreamer for EthBalanceEventStreamer { Err(e) => { log::error!("Failed getting addresses for {}. Error: {}", coin.ticker, e); let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); - ctx.stream_channel_controller - .broadcast(Event::err(streamer_id.clone(), e, None)) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; sleep_remaining_time(interval, now).await; continue; }, @@ -213,15 +222,13 @@ impl EventStreamer for EthBalanceEventStreamer { ); let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); // FIXME: We should add the address in the error message. - ctx.stream_channel_controller - .broadcast(Event::err(streamer_id.clone(), e, None)) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; }, }; } if !balance_updates.is_empty() { - ctx.stream_channel_controller + broadcaster .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) .await; } @@ -241,6 +248,6 @@ impl EventStreamer for EthBalanceEventStreamer { ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - start_polling(self.streamer_id(), self.coin, ctx, self.interval).await + start_polling(self.streamer_id(), broadcaster, self.coin, ctx, self.interval).await } } diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 05ea0cb8b4..01daf737d7 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -5,7 +5,7 @@ use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; @@ -38,7 +38,12 @@ impl EventStreamer for TendermintBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle( + self, + broadcaster: Controller, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); let coin = self.coin; @@ -144,9 +149,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { Err(e) => { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); - ctx.stream_channel_controller - .broadcast(Event::err(streamer_id.clone(), e, None)) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; continue; }, @@ -176,7 +179,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { } if !balance_updates.is_empty() { - ctx.stream_channel_controller + broadcaster .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) .await; } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index fe46426de9..f94a467b4b 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -4,7 +4,7 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, Filter, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, Filter, StreamHandlerInput}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -60,6 +60,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { async fn handle( self, + broadcaster: Controller, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { @@ -133,7 +134,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { Err(e) => { log::error!("{e}"); - ctx.stream_channel_controller + broadcaster .broadcast(Event::err( streamer_id.clone(), json!({ "error": e }), @@ -152,7 +153,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { Err(e) => { log::error!("{e}"); - ctx.stream_channel_controller + broadcaster .broadcast(Event::err( streamer_id.clone(), json!({ "error": e }), @@ -208,7 +209,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); - ctx.stream_channel_controller + broadcaster .broadcast(Event::err(streamer_id.clone(), e, Some(filter.clone()))) .await; @@ -222,7 +223,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); - ctx.stream_channel_controller + broadcaster .broadcast(Event::new( streamer_id.clone(), json!(vec![payload]), diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 86ee0f75d3..7bded69a70 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -10,7 +10,7 @@ use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, StreamHandlerInput}; use serde_json::Value as Json; use std::sync::Arc; @@ -41,7 +41,12 @@ impl EventStreamer for ZCoinBalanceEventStreamer { fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } - async fn handle(self, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput<()>) { + async fn handle( + self, + broadcaster: Controller, + ready_tx: oneshot::Sender>, + data_rx: impl StreamHandlerInput<()>, + ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); let coin = self.coin; @@ -82,7 +87,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); - ctx.stream_channel_controller + broadcaster .broadcast(Event::new(streamer_id.clone(), payload, None)) .await; }, @@ -90,10 +95,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { let ticker = coin.ticker(); error!("Failed getting balance for '{ticker}'. Error: {err}"); let e = serde_json::to_value(err).expect("Serialization should't fail."); - return ctx - .stream_channel_controller - .broadcast(Event::err(streamer_id.clone(), e, None)) - .await; + return broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; }, }; } diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index da64d621ac..26c6016cec 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -68,7 +68,7 @@ impl MmCtx { "data": data }); - self.stream_channel_controller + self.event_stream_manager .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input, None)) .await; diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 5bf9ea62a5..a95a4194f4 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -75,8 +75,6 @@ pub struct MmCtx { pub initialized: Constructible, /// True if the RPC HTTP server was started. pub rpc_started: Constructible, - /// Controller for continuously streaming data using streaming channels of `mm2_event_stream`. - pub stream_channel_controller: Controller, /// Data transfer bridge between server and client where server (which is the mm2 runtime) initiates the request. pub(crate) data_asker: DataAsker, /// Configuration of event streaming used for SSE. @@ -154,7 +152,6 @@ impl MmCtx { metrics: MetricsArc::new(), initialized: Constructible::default(), rpc_started: Constructible::default(), - stream_channel_controller: Controller::new(), data_asker: DataAsker::default(), event_stream_configuration: Default::default(), event_stream_manager: Default::default(), diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 338fdec74a..4f0b55bbea 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,7 +2,8 @@ use std::any::Any; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use super::EventStreamer; +use crate::{Controller, Event, EventStreamer}; + use common::executor::abortable_queue::WeakSpawner; use futures::channel::mpsc::UnboundedSender; @@ -22,6 +23,8 @@ pub enum StreamingSendError { pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: Arc, Option>>)>>>, + /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). + controller: Controller, } impl StreamingManager { @@ -31,7 +34,7 @@ impl StreamingManager { // NOTE: We spawn the streamer *before* checking if it can be added or not because // we don't know how long will it take for to spawn up and we don't want to lock // the manager for too long. - let channels = streamer.spawn(spawner).await?; + let channels = streamer.spawn(spawner, self.controller()).await?; let mut streamers = self.streamers.write().unwrap(); // If that streamer already exists, refuse to add it. if streamers.contains_key(&streamer_id) { @@ -62,4 +65,14 @@ impl StreamingManager { .ok_or(StreamingSendError::StreamerNotFound)?; Ok(()) } + + /// Broadcasts some event directly to listening clients. + /// + /// In contrast to `StreamingManager::send`, which sends some data to a streamer, + /// this method broadcasts an event to the listening *clients* directly, independently + /// of any streamer (i.e. bypassing any streamer). + pub async fn broadcast(&self, event: Event) { self.controller.broadcast(event).await } + + /// Returns the controller/broadcaster (the middle tie between the streamers and the clients). + pub fn controller(&self) -> Controller { self.controller.clone() } } diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 09812c0b58..6df97a0dff 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,5 +1,6 @@ use std::any::{self, Any}; +use crate::{Controller, Event}; use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; @@ -32,6 +33,7 @@ where /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. async fn handle( self, + broadcaster: Controller, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, ); @@ -48,6 +50,7 @@ where // we can use `self.spawner()` here to get it. // Also for AbortSettings, we can make this customizable with a default impl. spawner: WeakSpawner, + broadcaster: Controller, ) -> Result<(oneshot::Sender<()>, Option>>), String> { let streamer_id = self.streamer_id(); info!("Spawning event streamer: {streamer_id}"); @@ -81,7 +84,7 @@ where _ = rx_shutdown.fuse() => { info!("Manually shutting down event streamer: {streamer_id}.") } - _ = self.handle(tx_ready, data_receiver).fuse() => {} + _ = self.handle(broadcaster, tx_ready, data_receiver).fuse() => {} } } }; diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index b9a6ab6a2a..6e52b6e55f 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use serde::Deserialize; use serde_json::Value as Json; @@ -18,14 +18,12 @@ const fn default_stream_interval() -> f64 { 5. } pub struct HeartbeatEvent { config: HeartbeatEventConfig, - ctx: MmArc, } impl HeartbeatEvent { - pub fn try_new(config: Json, ctx: MmArc) -> Result { + pub fn try_new(config: Json) -> Result { Ok(Self { config: serde_json::from_value(config).map_err(|e| e.to_string())?, - ctx, }) } } @@ -36,12 +34,16 @@ impl EventStreamer for HeartbeatEvent { fn streamer_id(&self) -> String { "HEARTBEAT".to_string() } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle( + self, + broadcaster: Controller, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { ready_tx.send(Ok(())).unwrap(); loop { - self.ctx - .stream_channel_controller + broadcaster .broadcast(Event::new(self.streamer_id(), json!({}), None)) .await; diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 86a2e0b261..69fa4ea6a0 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -435,7 +435,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { } if let Some(config) = ctx.event_stream_configuration.get_event("HEARTBEAT") { - let heartbeat_streamer = HeartbeatEvent::try_new(config, ctx.clone()).map_to_mm(|e| { + let heartbeat_streamer = HeartbeatEvent::try_new(config).map_to_mm(|e| { MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")) })?; ctx.event_stream_manager diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 58d0987e41..ed96d3ef82 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; use serde_json::{json, Value as Json}; @@ -41,7 +41,12 @@ impl EventStreamer for NetworkEvent { fn streamer_id(&self) -> String { "NETWORK".to_string() } - async fn handle(self, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput) { + async fn handle( + self, + broadcaster: Controller, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { let p2p_ctx = P2PContext::fetch_from_mm_arc(&self.ctx); let mut previously_sent = json!({}); @@ -65,8 +70,7 @@ impl EventStreamer for NetworkEvent { }); if previously_sent != event_data || self.config.always_send { - self.ctx - .stream_channel_controller + broadcaster .broadcast(Event::new(self.streamer_id(), event_data.clone(), None)) .await; diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index 4ec8852f80..63f3d88081 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -29,7 +29,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Wed, 17 Jul 2024 11:08:50 +0200 Subject: [PATCH 029/128] fix wasm compilation --- mm2src/mm2_net/src/wasm_event_stream.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mm2src/mm2_net/src/wasm_event_stream.rs b/mm2src/mm2_net/src/wasm_event_stream.rs index a92134d6bc..7dbcd33e95 100644 --- a/mm2src/mm2_net/src/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/wasm_event_stream.rs @@ -1,5 +1,6 @@ use mm2_core::mm_ctx::MmArc; use serde_json::json; +use std::collections::HashSet; use web_sys::SharedWorker; struct SendableSharedWorker(SharedWorker); @@ -14,7 +15,7 @@ unsafe impl Send for SendableMessagePort {} pub async fn handle_worker_stream(ctx: MmArc) { let config = &ctx.event_stream_configuration; - let channel_controller = ctx.stream_channel_controller.clone(); + let channel_controller = ctx.event_stream_manager.controller(); let mut rx = channel_controller.create_channel(config.total_active_events()); let worker_path = config @@ -34,17 +35,21 @@ pub async fn handle_worker_stream(ctx: MmArc) { let port = SendableMessagePort(worker.0.port()); port.0.start(); + let filtered_events = HashSet::new(); while let Some(event) = rx.recv().await { - let data = json!({ - "_type": event.event_type(), - "message": event.message(), - }); - let message_js = wasm_bindgen::JsValue::from_str(&data.to_string()); + if let Some((event_type, message)) = event.get_data(&filtered_events) { + let data = json!({ + "_type": event_type, + "message": message, + }); - port.0.post_message(&message_js) + let message_js = wasm_bindgen::JsValue::from_str(&data.to_string()); + + port.0.post_message(&message_js) .expect("Failed to post a message to the SharedWorker.\n\ This could be due to the browser being incompatible.\n\ For more details, please refer to https://developer.mozilla.org/en-US/docs/Web/API/MessagePort/postMessage#browser_compatibility"); + } } } From 5655db683969e7603dffe4340a16fe74cc6c5ba2 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 11:22:02 +0200 Subject: [PATCH 030/128] use `fn handle`'s `data_rx` in zcoin balance event streaming this is still not hooked up though. no data will be received on this rx channel till we do `event_stream_manager.send('BALANCE:ZCoin') (or whatever the coin name is)` somewhere. --- mm2src/coins/z_coin/z_balance_streaming.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 7bded69a70..a1baf3f8fb 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -45,7 +45,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { self, broadcaster: Controller, ready_tx: oneshot::Sender>, - data_rx: impl StreamHandlerInput<()>, + mut data_rx: impl StreamHandlerInput<()>, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); @@ -68,17 +68,11 @@ impl EventStreamer for ZCoinBalanceEventStreamer { ready_tx, "MM context must have been initialized already." ); - let z_balance_change_handler = send_status_on_err!( - coin.z_fields.z_balance_event_handler.as_ref(), - ready_tx, - "Z balance change receiver can not be empty." - ); ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - // Locks the balance change handler, iterates through received events, and updates balance changes accordingly. - let mut bal = z_balance_change_handler.lock().await; - while (bal.next().await).is_some() { + // Iterates through received events, and updates balance changes accordingly. + while (data_rx.next().await).is_some() { match coin.my_balance().compat().await { Ok(balance) => { let payload = json!({ From 5a780ed7d6dfcf231d29b40f89be4deb0faff644 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 11:39:56 +0200 Subject: [PATCH 031/128] rename StreamingSendError -> StreamingManagerError also let StreamingManager::add return StreamingManagerError instead of String --- mm2src/coins/eth.rs | 2 +- .../utxo/utxo_builder/utxo_arc_builder.rs | 2 +- mm2src/coins/z_coin.rs | 2 +- .../src/tendermint_with_assets_activation.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 34 ++++++++++++------- mm2src/mm2_main/src/lp_native_dex.rs | 4 +-- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index dc93eb3f2b..c1da8a2d92 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5287,7 +5287,7 @@ impl EthCoin { ctx.event_stream_manager .add(balance_streamer, self.spawner()) .await - .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {}", e)) + .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {:?}", e)) } /// Requests the nonce from all available nodes and returns the highest nonce available with the list of nodes that returned the highest nonce. diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index 340d726bf4..d48efd17c9 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -127,7 +127,7 @@ where .add(utxo_streamer, utxo_arc.abortable_system.weak_spawner()) .await .map_to_mm(|e| { - UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to spawn utxo event streaming: {e}")) + UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to spawn utxo event streaming: {e:?}")) })?; Ok(result_coin) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 8624f47de0..8d65e3c8da 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -666,7 +666,7 @@ impl ZCoin { ctx.event_stream_manager .add(balance_streamer, self.spawner()) .await - .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {}", e)) + .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {:?}", e)) } } diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index d3363cab01..c8b0f887dd 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -391,7 +391,7 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { .map_to_mm(|e| TendermintInitError { ticker: self.ticker().to_owned(), kind: TendermintInitErrorKind::BalanceStreamInitError(format!( - "Failed to spawn tendermint balance streaming handler: {e}" + "Failed to spawn tendermint balance streaming handler: {e:?}" )), }) } diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 4f0b55bbea..eb229f488d 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -10,13 +10,16 @@ use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; /// The errors that could originate from the streaming manager. -pub enum StreamingSendError { +#[derive(Debug)] +pub enum StreamingManagerError { /// There is no streamer with the given ID. StreamerNotFound, /// Couldn't send the data to the streamer. SendError, /// The streamer doesn't accept an input. NoDataIn, + /// Couldn't spawn/add the streamer. + SpawnError(String), } #[derive(Clone, Default)] @@ -29,40 +32,45 @@ pub struct StreamingManager { impl StreamingManager { /// Spawns and adds a new streamer `streamer` to the manager. - pub async fn add(&self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), String> { + pub async fn add(&self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), StreamingManagerError> { let streamer_id = streamer.streamer_id(); // NOTE: We spawn the streamer *before* checking if it can be added or not because - // we don't know how long will it take for to spawn up and we don't want to lock - // the manager for too long. - let channels = streamer.spawn(spawner, self.controller()).await?; + // we don't know how long will it take to spawn up and we don't want to lock the + // manager for too long. + let channels = streamer + .spawn(spawner, self.controller()) + .await + .map_err(StreamingManagerError::SpawnError)?; let mut streamers = self.streamers.write().unwrap(); // If that streamer already exists, refuse to add it. if streamers.contains_key(&streamer_id) { - return Err(format!( + return Err(StreamingManagerError::SpawnError(format!( "A streamer with the same id ({streamer_id}) exists, it must be shutdown before re-using the same id." - )); + ))); } streamers.insert(streamer_id, channels); Ok(()) } /// Sends data to a streamer of a known ID. - pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingSendError> { + pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { let streamers = self.streamers.read().unwrap(); - let (_, data_in) = streamers.get(streamer_id).ok_or(StreamingSendError::StreamerNotFound)?; - let data_in = data_in.as_ref().ok_or(StreamingSendError::NoDataIn)?; + let (_, data_in) = streamers + .get(streamer_id) + .ok_or(StreamingManagerError::StreamerNotFound)?; + let data_in = data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; data_in .unbounded_send(Box::new(data)) - .map_err(|_| StreamingSendError::SendError) + .map_err(|_| StreamingManagerError::SendError) } /// Shuts down a streamer of a known ID. - pub fn shut(&self, streamer_id: &str) -> Result<(), StreamingSendError> { + pub fn shut(&self, streamer_id: &str) -> Result<(), StreamingManagerError> { self.streamers .write() .unwrap() .remove(streamer_id) - .ok_or(StreamingSendError::StreamerNotFound)?; + .ok_or(StreamingManagerError::StreamerNotFound)?; Ok(()) } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 69fa4ea6a0..e8aa9f8d3f 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -431,7 +431,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { ctx.event_stream_manager .add(network_steamer, ctx.spawner()) .await - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e}")))?; + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e:?}")))?; } if let Some(config) = ctx.event_stream_configuration.get_event("HEARTBEAT") { @@ -441,7 +441,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { ctx.event_stream_manager .add(heartbeat_streamer, ctx.spawner()) .await - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e}")))?; + .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e:?}")))?; } Ok(()) From a736f9b4da66ab189b9d39e54feb6475e7d598a2 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 11:45:33 +0200 Subject: [PATCH 032/128] remove context extraction in eth,zcoin,tendermint,utxo balance events we only extracted the ctx here to use the controller. since the controller is provided as an argument in EventStreamer::handle already, no need to extract the context in these senarios --- mm2src/coins/eth/eth_balance_events.rs | 20 ++----------------- .../tendermint/tendermint_balance_events.rs | 10 ---------- mm2src/coins/utxo/utxo_balance_events.rs | 10 ---------- mm2src/coins/z_coin/z_balance_streaming.rs | 7 ------- mm2src/mm2_core/src/mm_ctx.rs | 2 +- mm2src/mm2_main/src/heartbeat_event.rs | 1 - 6 files changed, 3 insertions(+), 47 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 4ad1f5d72e..f6c9d37df8 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -3,7 +3,6 @@ use common::{executor::Timer, log, Future01CompatExt}; use ethereum_types::Address; use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; -use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmError; use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; @@ -159,13 +158,7 @@ impl EventStreamer for EthBalanceEventStreamer { ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - async fn start_polling( - streamer_id: String, - broadcaster: Controller, - coin: EthCoin, - ctx: MmArc, - interval: f64, - ) { + async fn start_polling(streamer_id: String, broadcaster: Controller, coin: EthCoin, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, // our goal is to broadcast changed balances every x seconds. @@ -237,17 +230,8 @@ impl EventStreamer for EthBalanceEventStreamer { } } - let ctx = match MmArc::from_weak(&self.coin.ctx) { - Some(ctx) => ctx, - None => { - let msg = "MM context must have been initialized already."; - ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); - panic!("{}", msg); - }, - }; - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - start_polling(self.streamer_id(), broadcaster, self.coin, ctx, self.interval).await + start_polling(self.streamer_id(), broadcaster, self.coin, self.interval).await } } diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 01daf737d7..0a0b33db87 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -4,7 +4,6 @@ use futures::channel::oneshot; use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; -use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use serde_json::Value as Json; @@ -62,15 +61,6 @@ impl EventStreamer for TendermintBalanceEventStreamer { serde_json::to_string(&q).expect("This should never happen") } - let ctx = match MmArc::from_weak(&coin.ctx) { - Some(ctx) => ctx, - None => { - let msg = "MM context must have been initialized already."; - ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); - panic!("{}", msg); - }, - }; - let account_id = coin.account_id.to_string(); let mut current_balances: HashMap = HashMap::new(); diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index f94a467b4b..01c53a26a1 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -3,7 +3,6 @@ use common::{executor::Timer, log, Future01CompatExt}; use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; -use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Controller, Event, EventStreamer, Filter, StreamHandlerInput}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; @@ -113,15 +112,6 @@ impl EventStreamer for UtxoBalanceEventStreamer { panic!("Native RPC client is not supported for UtxoBalanceEventStreamer."); } - let ctx = match MmArc::from_weak(&coin.as_ref().ctx) { - Some(ctx) => ctx, - None => { - let msg = "MM context must have been initialized already."; - ready_tx.send(Err(msg.to_owned())).expect(RECEIVER_DROPPED_MSG); - panic!("{}", msg); - }, - }; - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); let mut scripthash_to_address_map = BTreeMap::default(); diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index a1baf3f8fb..9e1007f529 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -9,7 +9,6 @@ use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; -use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Controller, Event, EventStreamer, StreamHandlerInput}; use serde_json::Value as Json; use std::sync::Arc; @@ -63,12 +62,6 @@ impl EventStreamer for ZCoinBalanceEventStreamer { }; } - let ctx = send_status_on_err!( - MmArc::from_weak(&coin.as_ref().ctx), - ready_tx, - "MM context must have been initialized already." - ); - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); // Iterates through received events, and updates balance changes accordingly. diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index a95a4194f4..728337809d 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{manager::StreamingManager, Controller, Event, EventStreamConfiguration}; +use mm2_event_stream::{manager::StreamingManager, EventStreamConfiguration}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 6e52b6e55f..3f27ced811 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -1,7 +1,6 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; -use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use serde::Deserialize; use serde_json::Value as Json; From eea5e026853eeff12dce71881bd91bd3405c7cd7 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 11:47:16 +0200 Subject: [PATCH 033/128] no need for error handling macro in zcoin balance streaming looks like there are no more errors that could arise :/ --- mm2src/coins/z_coin/z_balance_streaming.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 9e1007f529..6d7a6bac0d 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -46,23 +46,12 @@ impl EventStreamer for ZCoinBalanceEventStreamer { ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput<()>, ) { - const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); let coin = self.coin; - macro_rules! send_status_on_err { - ($match: expr, $sender: tt, $msg: literal) => { - match $match { - Some(t) => t, - None => { - $sender.send(Err($msg.to_owned())).expect(RECEIVER_DROPPED_MSG); - panic!("{}", $msg); - }, - } - }; - } - - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); // Iterates through received events, and updates balance changes accordingly. while (data_rx.next().await).is_some() { From 98f9253bef5e131aea104bbf94f67544154323f7 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 11:52:33 +0200 Subject: [PATCH 034/128] remove ctx from UtxoCoinFields (for now) no need for the feild. looks like the only use for it was to access the channel controller. --- mm2src/coins/utxo.rs | 3 +-- mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs | 2 -- mm2src/coins/utxo/utxo_common_tests.rs | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index da5e640e23..40e77dcf6b 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -67,7 +67,7 @@ pub use keys::{Address, AddressBuilder, AddressFormat as UtxoAddressFormat, Addr AddressScriptType, KeyPair, LegacyAddress, Private, Public, Secret}; #[cfg(not(target_arch = "wasm32"))] use lightning_invoice::Currency as LightningCurrency; -use mm2_core::mm_ctx::{MmArc, MmWeak}; +use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; @@ -618,7 +618,6 @@ pub struct UtxoCoinFields { /// This abortable system is used to spawn coin's related futures that should be aborted on coin deactivation /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, - pub(crate) ctx: MmWeak, /// This is used for balance event streaming implementation for UTXOs. /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, /// it will be used for receiving scripthash notifications to re-fetch balances. diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 2cbb57b35c..2578b4d13a 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -305,7 +305,6 @@ where block_headers_status_watcher, abortable_system, scripthash_notification_handler, - ctx: builder.ctx().weak(), }; Ok(coin) @@ -391,7 +390,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { block_headers_status_watcher, abortable_system, scripthash_notification_handler, - ctx: self.ctx().weak(), }; Ok(coin) } diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index 03fea53699..dde8ddd43a 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -149,7 +149,6 @@ pub(super) fn utxo_coin_fields_for_test( block_headers_status_watcher: None, abortable_system: AbortableQueue::default(), scripthash_notification_handler: None, - ctx: Default::default(), } } From 08401d3ea884db1ab1464c5a37263e62dff43479 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 17 Jul 2024 13:38:22 +0200 Subject: [PATCH 035/128] clear filtered events in sse_handler we want to use another type of filter (to enable and disable streams dynamically while keeping the SSE connection open) --- mm2src/mm2_net/src/sse_handler.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index 63f3d88081..20d8f2de0b 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -15,19 +15,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Fri, 19 Jul 2024 07:05:09 +0200 Subject: [PATCH 038/128] remove eventname --- mm2src/mm2_event_stream/src/event.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index 76fd106cdb..8ddcad2652 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -71,24 +71,3 @@ pub trait Filter: Send + Sync { /// `requested_events` is a set of the events that the client asked to subscribe to (e.g. `BALANCE:BTC`) fn filter(&self, message: &Json, requested_events: &HashSet) -> Option; } - -/// Event types streamed to clients through channels like Server-Sent Events (SSE). -#[derive(Deserialize, Eq, Hash, PartialEq)] -pub enum EventName { - /// Indicates a change in the balance of a coin. - BALANCE, - /// Event triggered at regular intervals to indicate that the system is operational. - HEARTBEAT, - /// Returns p2p network information at a regular interval. - NETWORK, -} - -impl fmt::Display for EventName { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::BALANCE => write!(f, "COIN_BALANCE"), - Self::HEARTBEAT => write!(f, "HEARTBEAT"), - Self::NETWORK => write!(f, "NETWORK"), - } - } -} From 67348ab0e41fc63c21f7a987ec6786a10ee1a1f7 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 07:11:10 +0200 Subject: [PATCH 039/128] keep streamer id for both normal and error events the same change it only when extracting the output out --- mm2src/mm2_event_stream/src/event.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index 8ddcad2652..d89fde1eb3 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -8,7 +8,7 @@ use serde_json::Value as Json; /// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { /// The type of the event (balance, network, swap, etc...). - event_type: String, + streamer_id: String, /// The message to be sent to the client. message: Json, /// The filter object to be used to determine whether the event should be sent or not. @@ -17,29 +17,42 @@ pub struct Event { /// The filter is wrapped in an `Arc` since the event producer should use it as a singleton /// by using the same filter over and over again with multiple events. filter: Option>, + /// Indicating whether this event is an error event or a normal one. + error: bool, } impl Event { /// Creates a new `Event` instance with the specified event type and message. #[inline] - pub fn new(event_type: String, message: Json, filter: Option>) -> Self { + pub fn new(streamer_id: String, message: Json, filter: Option>) -> Self { Self { - event_type, + streamer_id, message, filter, + error: false, } } /// Create a new error `Event` instance with the specified error event type and message. #[inline] - pub fn err(event_type: String, message: Json, filter: Option>) -> Self { + pub fn err(streamer_id: String, message: Json, filter: Option>) -> Self { Self { - event_type: format!("ERROR_{event_type}"), + streamer_id, message, filter, + error: true, } } + pub fn origin(&self) -> &str { + &self.streamer_id + } + + pub fn get(&self) -> (String, Json) { + let prefix = if self.error { "ERROR:" } else { "" }; + (format!("{prefix}{streamer_id}"), self.message.clone()) + } + /// Returns the event type and message to be sent or `None` if the event should not be sent. /// /// Uses the `requested_events` to determine whether the event should be sent or not. @@ -50,11 +63,11 @@ impl Event { pub fn get_data(&self, requested_events: &HashSet) -> Option<(String, Json)> { self.filter.as_ref().map_or_else( // If no filter is set, send the event as is. - || Some((self.event_type.clone(), self.message.clone())), + || Some((self.streamer_id.clone(), self.message.clone())), |filter| { filter .filter(&self.message, requested_events) - .map(|message| (self.event_type.clone(), message)) + .map(|message| (self.streamer_id.clone(), message)) }, ) } From aab0b1c3028fea4a69064b2bde78431fc9bf6507 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 07:49:10 +0200 Subject: [PATCH 040/128] let the controller broadcast only to certain clients instead of applying the filter at event level, we will instead filter while broadcasting. in here we choose which clients we want the event to be broadcasted to. This should also reduce unnecessary overhead for slow client connections who don't pick up the events quickly (they now won't receive unrelated events, thus keeping extra more space in their channels) --- mm2src/mm2_event_stream/src/controller.rs | 24 ++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 88284b97a8..49df3167f1 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -1,5 +1,6 @@ use parking_lot::Mutex; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::{HashMap, HashSet}, + sync::Arc}; use tokio::sync::mpsc::{self, Receiver, Sender}; type ChannelId = u64; @@ -53,9 +54,9 @@ impl Controller { pub fn num_connections(&self) -> usize { self.0.lock().channels.len() } /// Broadcast message to all channels - pub async fn broadcast(&self, message: M) { + pub async fn broadcast(&self, message: M, client_ids: Option>>) { let msg = Arc::new(message); - for tx in self.all_channels() { + for tx in self.channels(client_ids) { // Only `try_send` here. If the receiver's channel is full (receiver is slow), it will // not receive the message. This avoids blocking the broadcast to other receivers. tx.try_send(msg.clone()).ok(); @@ -68,8 +69,21 @@ impl Controller { inner.channels.remove(channel_id); } - /// Returns all the active channel senders - fn all_channels(&self) -> Vec>> { self.0.lock().channels.values().cloned().collect() } + /// Returns the channels that are associated with the specified client ids. + /// + /// If no client ids are specified, all the channels are returned. + fn channels(&self, client_ids: Option>>) -> Vec>> { + if let Some(client_ids) = client_ids { + self.0 + .lock() + .channels + .iter() + .filter_map(|(id, sender)| client_ids.contains(value).then(sender.clone())); + } else { + // Returns all the channels if no client ids where specifically requested. + self.0.lock().channels.values().cloned().collect() + } + } } impl Default for Controller { From 620e35e0be28c5025d50728e81814ab795ed4301 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 10:50:26 +0200 Subject: [PATCH 041/128] convert to sub/pub arch now clients will subscribe to some streamer. if the streamer doesn't yet exist, it will be spawned using the client provided configuration. and the client will be registered with as a listener to that streamer (so every new event from that streamer will be sent to that client). on the other hand, if the streamer already exists and a client is trying to add/spawn it, the client will simply be registered to the alraedy runnning streamer and no changes to the streamer configuration is made (even if the clinet's wanted config is different). now new events from this streamer will also be sent to the newly added client. a client can shut/(stop listening to) a streamer, in this case, if the streamer has no more clients to server, it will be terminated. otherwise, it's kept alive to sever other clients. a clinet can also terminate the sse connection (remove_client), in which case all of its streamers are shut (same as the shut point above) --- mm2src/mm2_event_stream/src/controller.rs | 14 +- mm2src/mm2_event_stream/src/event.rs | 6 +- mm2src/mm2_event_stream/src/lib.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 160 +++++++++++++++++----- mm2src/mm2_event_stream/src/streamer.rs | 5 +- 5 files changed, 140 insertions(+), 47 deletions(-) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 49df3167f1..d068b6374d 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -14,7 +14,6 @@ impl Clone for Controller { /// Inner part of the controller pub struct ChannelsInner { - last_id: u64, channels: HashMap>>, } @@ -31,13 +30,11 @@ impl Controller { pub fn new() -> Self { Default::default() } /// Creates a new channel and returns it's events receiver - pub fn create_channel(&self, concurrency: usize) -> ChannelReceiver { + pub fn create_channel(&self, channel_id: ChannelId, concurrency: usize) -> ChannelReceiver { let (tx, rx) = mpsc::channel::>(concurrency); let mut inner = self.0.lock(); - let channel_id = inner.last_id.overflowing_add(1).0; inner.channels.insert(channel_id, tx); - inner.last_id = channel_id; let channel_drop_hook = { let controller = self.clone(); @@ -54,7 +51,7 @@ impl Controller { pub fn num_connections(&self) -> usize { self.0.lock().channels.len() } /// Broadcast message to all channels - pub async fn broadcast(&self, message: M, client_ids: Option>>) { + pub fn broadcast(&self, message: M, client_ids: Option<&HashSet>) { let msg = Arc::new(message); for tx in self.channels(client_ids) { // Only `try_send` here. If the receiver's channel is full (receiver is slow), it will @@ -72,13 +69,15 @@ impl Controller { /// Returns the channels that are associated with the specified client ids. /// /// If no client ids are specified, all the channels are returned. - fn channels(&self, client_ids: Option>>) -> Vec>> { + fn channels(&self, client_ids: Option<&HashSet>) -> Vec>> { if let Some(client_ids) = client_ids { self.0 .lock() .channels .iter() - .filter_map(|(id, sender)| client_ids.contains(value).then(sender.clone())); + .filter_map(|(id, sender)| client_ids.contains(id).then_some(sender)) + .cloned() + .collect() } else { // Returns all the channels if no client ids where specifically requested. self.0.lock().channels.values().cloned().collect() @@ -89,7 +88,6 @@ impl Controller { impl Default for Controller { fn default() -> Self { let inner = ChannelsInner { - last_id: 0, channels: HashMap::new(), }; Self(Arc::new(Mutex::new(inner))) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index d89fde1eb3..ada42ddfdd 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -44,13 +44,11 @@ impl Event { } } - pub fn origin(&self) -> &str { - &self.streamer_id - } + pub fn origin(&self) -> &str { &self.streamer_id } pub fn get(&self) -> (String, Json) { let prefix = if self.error { "ERROR:" } else { "" }; - (format!("{prefix}{streamer_id}"), self.message.clone()) + (format!("{prefix}{}", self.streamer_id), self.message.clone()) } /// Returns the event type and message to be sent or `None` if the event should not be sent. diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index ddec11d74e..9c7a17dc1f 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -7,5 +7,5 @@ pub mod streamer; // Re-export important types. pub use configuration::EventStreamConfiguration; pub use controller::Controller; -pub use event::{Event, EventName, Filter}; +pub use event::{Event, Filter}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index be44779d10..a77e683bcd 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,7 +1,8 @@ use std::any::Any; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; +use crate::controller::ChannelReceiver; use crate::{Controller, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; @@ -20,60 +21,116 @@ pub enum StreamingManagerError { NoDataIn, /// Couldn't spawn the streamer. SpawnError(String), - /// Couldn't add the streamer, because it already exists. - AddError(String), + /// The client is not known/registered. + UnknownClient(u64), + /// A client with the same ID already exists. + ClientExists(u64), + /// The client is already listening to the streamer. + ClientAlreadyListening(u64, String), +} + +struct StreamerInfo { + /// The communication channel to the streamer. + data_in: Option>>, + /// Clients the streamer is serving for. + clients: HashSet, + /// The shutdown handle of the streamer. + _shutdown: oneshot::Sender<()>, +} + +impl StreamerInfo { + fn new(data_in: Option>>, shutdown: oneshot::Sender<()>) -> Self { + Self { + data_in, + clients: HashSet::new(), + _shutdown: shutdown, + } + } } #[derive(Clone, Default)] pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. - streamers: Arc, Option>>)>>>, + streamers: Arc>>, /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). controller: Controller, + /// An inverse map from clients to the streamers they are listening to. + clients: Arc>>>, } impl StreamingManager { /// Spawns and adds a new streamer `streamer` to the manager. - pub async fn add(&self, streamer: impl EventStreamer, spawner: WeakSpawner) -> Result<(), StreamingManagerError> { + pub async fn add( + &self, + client_id: u64, + streamer: impl EventStreamer, + spawner: WeakSpawner, + ) -> Result<(), StreamingManagerError> { + // Make sure we know this client. + if !self.clients.read().unwrap().contains_key(&client_id) { + return Err(StreamingManagerError::UnknownClient(client_id)); + } let streamer_id = streamer.streamer_id(); - // NOTE: We spawn the streamer *before* checking if it can be added or not because - // we don't know how long will it take to spawn up and we don't want to lock the - // manager for too long. - let channels = streamer - .spawn(spawner, self.controller()) + // If a streamer is already up and running, add the client to it and return. + if let Some(streamer_info) = self.streamers.write().unwrap().get_mut(&streamer_id) { + if streamer_info.clients.contains(&client_id) { + // FIXME: such info could have been known from `client_to_streamers` map + // check the atomicity of these operations (considering we are running inside an async context). + return Err(StreamingManagerError::ClientAlreadyListening(client_id, streamer_id)); + } + streamer_info.clients.insert(client_id); + // FIXME: The streamer running is probably running with different configuration. + // We might want to inform the client that the configuration they asked for wasn't + // applied and return the active configuration instead? + // Note that there is no way to (re)spawn a streamer with a different configuration + // unless then we want to have different streamer for each and every client! + return Ok(()); + } + // Otherwise spawn a new streamer. + let (shutdown, data_in) = streamer + .spawn(spawner, self.clone()) .await .map_err(StreamingManagerError::SpawnError)?; - let mut streamers = self.streamers.write().unwrap(); - // If that streamer already exists, refuse to add it. - if streamers.contains_key(&streamer_id) { - return Err(StreamingManagerError::AddError(format!( - "A streamer with the same id ({streamer_id}) exists, it must be shutdown before re-using the same id." - ))); - } - streamers.insert(streamer_id, channels); + let streamer_info = StreamerInfo::new(data_in, shutdown); + // Note that while we were spawning the streamer (potentially a long operation) and not holding the lock + // to ths streamers map (and we indeed shouldn't), another client might have added the same streamer. + // So we shouldn't assume that the streamer doesn't exist based solely on the check above. + self.streamers + .write() + .unwrap() + .entry(streamer_id.clone()) + .or_insert(streamer_info) + .clients + .insert(client_id); + self.clients + .write() + .unwrap() + .entry(client_id) + .or_insert(HashSet::new()) + .insert(streamer_id); Ok(()) } - /// Sends data to a streamer of a known ID. + /// Sends data to a streamer with `streamer_id`. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { let streamers = self.streamers.read().unwrap(); - let (_, data_in) = streamers + let streamer_info = streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; - let data_in = data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; + let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; data_in .unbounded_send(Box::new(data)) .map_err(|_| StreamingManagerError::SendError) } - /// Shuts down a streamer of a known ID. - pub fn shut(&self, streamer_id: &str) -> Result<(), StreamingManagerError> { - self.streamers - .write() - .unwrap() - .remove(streamer_id) - .ok_or(StreamingManagerError::StreamerNotFound)?; - Ok(()) + /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. + pub fn stop(&self, client_id: u64, streamer_id: &str) { + self.streamers.write().unwrap().get_mut(streamer_id).map(|info| { + info.clients.remove(&client_id); + }); + self.clients.write().unwrap().get_mut(&client_id).map(|streamer_ids| { + streamer_ids.remove(streamer_id); + }); } /// Broadcasts some event directly to listening clients. @@ -81,8 +138,47 @@ impl StreamingManager { /// In contrast to `StreamingManager::send`, which sends some data to a streamer, /// this method broadcasts an event to the listening *clients* directly, independently /// of any streamer (i.e. bypassing any streamer). - pub async fn broadcast(&self, event: Event) { self.controller.broadcast(event).await } + pub fn broadcast(&self, event: Event) { + if let Some(client_ids) = self + .streamers + .read() + .unwrap() + .get(event.origin()) + .map(|info| &info.clients) + { + self.controller.broadcast(event, Some(client_ids)) + } + } + + /// Forcefully broadcasts an event to all clients even if they are not listening for such an event. + pub fn broadcast_all(&self, event: Event) { self.controller.broadcast(event, None) } + + pub fn new_client(&self, client_id: u64) -> Result, StreamingManagerError> { + let mut clients = self.clients.write().unwrap(); + if clients.contains_key(&client_id) { + return Err(StreamingManagerError::ClientExists(client_id)); + } + clients.insert(client_id, HashSet::new()); + Ok(self.controller.create_channel(client_id, 1024)) + } - /// Returns the controller/broadcaster (the middle tie between the streamers and the clients). - pub fn controller(&self) -> Controller { self.controller.clone() } + pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { + let streamer_ids = self + .clients + .write() + .unwrap() + .remove(&client_id) + .ok_or(StreamingManagerError::UnknownClient(client_id))?; + let mut streamers = self.streamers.write().unwrap(); + for streamer_id in streamer_ids { + if let Some(streamer_info) = streamers.get_mut(&streamer_id) { + streamer_info.clients.remove(&client_id); + }; + // If there are no more listening clients, terminate the streamer. + if streamers.get(&streamer_id).map(|info| info.clients.len()) == Some(0) { + streamers.remove(&streamer_id); + } + } + Ok(()) + } } diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 6df97a0dff..499738cab0 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,5 +1,6 @@ use std::any::{self, Any}; +use crate::manager::StreamingManager; use crate::{Controller, Event}; use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; @@ -33,7 +34,7 @@ where /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, ); @@ -50,7 +51,7 @@ where // we can use `self.spawner()` here to get it. // Also for AbortSettings, we can make this customizable with a default impl. spawner: WeakSpawner, - broadcaster: Controller, + broadcaster: StreamingManager, ) -> Result<(oneshot::Sender<()>, Option>>), String> { let streamer_id = self.streamer_id(); info!("Spawning event streamer: {streamer_id}"); From 571b1a4f5a9354351a9ef5d18a07f31b738a2d44 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 12:18:02 +0200 Subject: [PATCH 042/128] move the arc(Arc>>); - -impl Clone for Controller { - fn clone(&self) -> Self { Self(Arc::clone(&self.0)) } -} - -/// Inner part of the controller -pub struct ChannelsInner { - channels: HashMap>>, -} - -/// A smart channel receiver that will run a hook when it's dropped to remove itself from the controller. -pub struct ChannelReceiver { - /// The receiver end of the channel. - rx: Receiver>, - /// A hook that's ran when the receiver channel receiver is dropped. - channel_drop_hook: Box, +pub struct Controller { + channels: HashMap>>, } impl Controller { /// Creates a new channels controller - pub fn new() -> Self { Default::default() } + pub fn new() -> Self { + Self { + channels: HashMap::new(), + } + } /// Creates a new channel and returns it's events receiver - pub fn create_channel(&self, channel_id: ChannelId, concurrency: usize) -> ChannelReceiver { - let (tx, rx) = mpsc::channel::>(concurrency); - - let mut inner = self.0.lock(); - inner.channels.insert(channel_id, tx); - - let channel_drop_hook = { - let controller = self.clone(); - // Remove the channel from the controller when the receiver is dropped. - Box::new(move || { - common::log::debug!("Dropping event receiver"); - controller.remove_channel(&channel_id) - }) - }; - ChannelReceiver { rx, channel_drop_hook } + pub fn create_channel(&mut self, channel_id: u64, concurrency: usize) -> Receiver> { + let (tx, rx) = mpsc::channel(concurrency); + self.channels.insert(channel_id, tx); + rx } + /// Removes the channel from the controller + pub fn remove_channel(&mut self, channel_id: &u64) { self.channels.remove(channel_id); } + /// Returns number of active channels - pub fn num_connections(&self) -> usize { self.0.lock().channels.len() } + pub fn num_connections(&self) -> usize { self.channels.len() } /// Broadcast message to all channels pub fn broadcast(&self, message: M, client_ids: Option<&HashSet>) { @@ -60,49 +40,23 @@ impl Controller { } } - /// Removes the channel from the controller - fn remove_channel(&self, channel_id: &ChannelId) { - let mut inner = self.0.lock(); - inner.channels.remove(channel_id); - } - /// Returns the channels that are associated with the specified client ids. /// /// If no client ids are specified, all the channels are returned. fn channels(&self, client_ids: Option<&HashSet>) -> Vec>> { if let Some(client_ids) = client_ids { - self.0 - .lock() - .channels + self.channels .iter() .filter_map(|(id, sender)| client_ids.contains(id).then_some(sender)) .cloned() .collect() } else { // Returns all the channels if no client ids where specifically requested. - self.0.lock().channels.values().cloned().collect() + self.channels.values().cloned().collect() } } } -impl Default for Controller { - fn default() -> Self { - let inner = ChannelsInner { - channels: HashMap::new(), - }; - Self(Arc::new(Mutex::new(inner))) - } -} - -impl ChannelReceiver { - /// Receives the next event from the channel - pub async fn recv(&mut self) -> Option> { self.rx.recv().await } -} - -impl Drop for ChannelReceiver { - fn drop(&mut self) { (self.channel_drop_hook)(); } -} - #[cfg(any(test, target_arch = "wasm32"))] mod tests { use super::*; diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index ada42ddfdd..e45852f9b4 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use serde::Deserialize; use serde_json::Value as Json; +#[derive(Default)] /// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { /// The type of the event (balance, network, swap, etc...). diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index a77e683bcd..527d23dd7c 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,13 +2,12 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; -use crate::controller::ChannelReceiver; use crate::{Controller, Event, EventStreamer}; - use common::executor::abortable_queue::WeakSpawner; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; +use tokio::sync::mpsc::Receiver; /// The errors that could originate from the streaming manager. #[derive(Debug)] @@ -53,7 +52,7 @@ pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: Arc>>, /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). - controller: Controller, + controller: Arc>>, /// An inverse map from clients to the streamers they are listening to. clients: Arc>>>, } @@ -125,12 +124,17 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) { - self.streamers.write().unwrap().get_mut(streamer_id).map(|info| { - info.clients.remove(&client_id); - }); self.clients.write().unwrap().get_mut(&client_id).map(|streamer_ids| { streamer_ids.remove(streamer_id); }); + let mut streamers = self.streamers.write().unwrap(); + streamers.get_mut(streamer_id).map(|info| { + info.clients.remove(&client_id); + }); + // If there are no more listening clients, terminate the streamer. + if streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { + streamers.remove(streamer_id); + } } /// Broadcasts some event directly to listening clients. @@ -146,20 +150,22 @@ impl StreamingManager { .get(event.origin()) .map(|info| &info.clients) { - self.controller.broadcast(event, Some(client_ids)) + self.controller.read().unwrap().broadcast(event, Some(client_ids)) } } /// Forcefully broadcasts an event to all clients even if they are not listening for such an event. - pub fn broadcast_all(&self, event: Event) { self.controller.broadcast(event, None) } + pub fn broadcast_all(&self, event: Event) { self.controller.read().unwrap().broadcast(event, None) } - pub fn new_client(&self, client_id: u64) -> Result, StreamingManagerError> { + pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { let mut clients = self.clients.write().unwrap(); if clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists(client_id)); } clients.insert(client_id, HashSet::new()); - Ok(self.controller.create_channel(client_id, 1024)) + // Note that events queued in the channel are `Arc` shared. + // So 1024 long buffer won't be heavy on memory. + Ok(self.controller.write().unwrap().create_channel(client_id, 1024)) } pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { @@ -169,6 +175,8 @@ impl StreamingManager { .unwrap() .remove(&client_id) .ok_or(StreamingManagerError::UnknownClient(client_id))?; + self.controller.write().unwrap().remove_channel(&client_id); + let mut streamers = self.streamers.write().unwrap(); for streamer_id in streamer_ids { if let Some(streamer_info) = streamers.get_mut(&streamer_id) { From 0ed45e42f8e7ea30f90b1094374de1653b556973 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 12:48:30 +0200 Subject: [PATCH 043/128] wrap the clients hashmap with a mutex instead of rwlock --- mm2src/mm2_event_stream/src/manager.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 527d23dd7c..4e5ff725f0 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,6 +1,6 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use crate::{Controller, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; @@ -50,11 +50,13 @@ impl StreamerInfo { #[derive(Clone, Default)] pub struct StreamingManager { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. + // Using an RwLock assuming `StreamingManager::send` will be heavily used. streamers: Arc>>, /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). + // Using an RwLock assuming `StreamingManager::broadcast` will be heavily used. controller: Arc>>, /// An inverse map from clients to the streamers they are listening to. - clients: Arc>>>, + clients: Arc>>>, } impl StreamingManager { @@ -66,7 +68,7 @@ impl StreamingManager { spawner: WeakSpawner, ) -> Result<(), StreamingManagerError> { // Make sure we know this client. - if !self.clients.read().unwrap().contains_key(&client_id) { + if !self.clients.lock().unwrap().contains_key(&client_id) { return Err(StreamingManagerError::UnknownClient(client_id)); } let streamer_id = streamer.streamer_id(); @@ -102,7 +104,7 @@ impl StreamingManager { .clients .insert(client_id); self.clients - .write() + .lock() .unwrap() .entry(client_id) .or_insert(HashSet::new()) @@ -124,7 +126,7 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) { - self.clients.write().unwrap().get_mut(&client_id).map(|streamer_ids| { + self.clients.lock().unwrap().get_mut(&client_id).map(|streamer_ids| { streamer_ids.remove(streamer_id); }); let mut streamers = self.streamers.write().unwrap(); @@ -158,7 +160,7 @@ impl StreamingManager { pub fn broadcast_all(&self, event: Event) { self.controller.read().unwrap().broadcast(event, None) } pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { - let mut clients = self.clients.write().unwrap(); + let mut clients = self.clients.lock().unwrap(); if clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists(client_id)); } @@ -171,7 +173,7 @@ impl StreamingManager { pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { let streamer_ids = self .clients - .write() + .lock() .unwrap() .remove(&client_id) .ok_or(StreamingManagerError::UnknownClient(client_id))?; From 94621f3b23880ac4380ce982f8267fdb292175c0 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 14:55:43 +0200 Subject: [PATCH 044/128] ensure atomicity in streaming manager operations This refactor makes sure that we perform write operations on the streaming manager atomically. We were previouly locking some fields of the manager to edit them then unlock them and lock another field. This clearly isn't atomic since there is a depedency between the data stored in the two fields and they should be set in one step with no operation in between. One solution could have been to just wrap the whole manager with a mutex and not allow concurrent access to it. This would hurt the read perf though, which is critical for frequent send calls and broadcast calls. Another would have been introducing a new field (mutex) called `critical` or something and lock it when we want no other writing method to execute in parallel. This is the approach taken here, but since `clinets` field is already locked with every operation (every operation mutating the state of the streaming manager must involve a client anyway: added, removed, newly listening, etc...), we now lock the `clients` field before any cirtical section till the end so we are sure no other mutating code will run in parallel. --- mm2src/mm2_event_stream/src/manager.rs | 156 +++++++++++++++---------- 1 file changed, 94 insertions(+), 62 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 4e5ff725f0..36591f80bc 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -4,6 +4,7 @@ use std::sync::{Arc, Mutex, RwLock}; use crate::{Controller, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; +use common::log; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; @@ -21,11 +22,11 @@ pub enum StreamingManagerError { /// Couldn't spawn the streamer. SpawnError(String), /// The client is not known/registered. - UnknownClient(u64), + UnknownClient, /// A client with the same ID already exists. - ClientExists(u64), + ClientExists, /// The client is already listening to the streamer. - ClientAlreadyListening(u64, String), + ClientAlreadyListening, } struct StreamerInfo { @@ -56,6 +57,9 @@ pub struct StreamingManager { // Using an RwLock assuming `StreamingManager::broadcast` will be heavily used. controller: Arc>>, /// An inverse map from clients to the streamers they are listening to. + /// Lists known clients and is always kept in sync with `streamers`. + // NOTE: Any operation writing to this field will also write to `streamers` and potentially `controller`. + // Make sure to lock this field first and all the way until the changes are made to other fields. clients: Arc>>>, } @@ -67,48 +71,66 @@ impl StreamingManager { streamer: impl EventStreamer, spawner: WeakSpawner, ) -> Result<(), StreamingManagerError> { - // Make sure we know this client. - if !self.clients.lock().unwrap().contains_key(&client_id) { - return Err(StreamingManagerError::UnknownClient(client_id)); - } let streamer_id = streamer.streamer_id(); - // If a streamer is already up and running, add the client to it and return. - if let Some(streamer_info) = self.streamers.write().unwrap().get_mut(&streamer_id) { - if streamer_info.clients.contains(&client_id) { - // FIXME: such info could have been known from `client_to_streamers` map - // check the atomicity of these operations (considering we are running inside an async context). - return Err(StreamingManagerError::ClientAlreadyListening(client_id, streamer_id)); + + // Pre-checks before spawning the streamer. Inside another scope to drop the locks early. + { + let mut clients = self.clients.lock().unwrap(); + match clients.get(&client_id) { + // We don't know that client. We don't have a connection to it. + None => return Err(StreamingManagerError::UnknownClient), + // The client is already listening to that streamer. + Some(listening_to) if listening_to.contains(&streamer_id) => { + return Err(StreamingManagerError::ClientAlreadyListening); + }, + _ => (), + } + // If a streamer is already up and running, we won't spawn another one. + if let Some(streamer_info) = self.streamers.write().unwrap().get_mut(&streamer_id) { + // Register the client as a listener to the streamer. + streamer_info.clients.insert(client_id); + // Register the streamer as listened-to by the client. + clients + .get_mut(&client_id) + .map(|listening_to| listening_to.insert(streamer_id)); + // FIXME: The streamer running is probably running with different configuration. + // We might want to inform the client that the configuration they asked for wasn't + // applied and return the active configuration instead? + // + // Note that there is no way to (re)spawn a streamer with a different configuration + // unless then we want to have different streamer for each and every client! + // + // Another restricted solution is to not let clients set the streamer configuration + // and only allow it through MM2 configuration file. + return Ok(()); } - streamer_info.clients.insert(client_id); - // FIXME: The streamer running is probably running with different configuration. - // We might want to inform the client that the configuration they asked for wasn't - // applied and return the active configuration instead? - // Note that there is no way to (re)spawn a streamer with a different configuration - // unless then we want to have different streamer for each and every client! - return Ok(()); } - // Otherwise spawn a new streamer. + + // Spawn a new streamer. let (shutdown, data_in) = streamer .spawn(spawner, self.clone()) .await .map_err(StreamingManagerError::SpawnError)?; let streamer_info = StreamerInfo::new(data_in, shutdown); - // Note that while we were spawning the streamer (potentially a long operation) and not holding the lock - // to ths streamers map (and we indeed shouldn't), another client might have added the same streamer. - // So we shouldn't assume that the streamer doesn't exist based solely on the check above. - self.streamers - .write() - .unwrap() - .entry(streamer_id.clone()) - .or_insert(streamer_info) - .clients - .insert(client_id); - self.clients - .lock() - .unwrap() - .entry(client_id) - .or_insert(HashSet::new()) - .insert(streamer_id); + + // Note that we didn't hold the locks while spawning the streamer (potentially a long operation). + // This means we can't assume either that the client still exists at this point or + // that the streamer still doesn't exist. + let mut clients = self.clients.lock().unwrap(); + if let Some(listening_to) = clients.get_mut(&client_id) { + listening_to.insert(streamer_id.clone()); + self.streamers + .write() + .unwrap() + .entry(streamer_id.clone()) + .or_insert(streamer_info) + .clients + .insert(client_id); + } else { + // The client was removed while we were spawning the streamer. + // We no longer have a connection for it. + return Err(StreamingManagerError::UnknownClient); + } Ok(()) } @@ -125,21 +147,29 @@ impl StreamingManager { } /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. - pub fn stop(&self, client_id: u64, streamer_id: &str) { - self.clients.lock().unwrap().get_mut(&client_id).map(|streamer_ids| { - streamer_ids.remove(streamer_id); - }); - let mut streamers = self.streamers.write().unwrap(); - streamers.get_mut(streamer_id).map(|info| { - info.clients.remove(&client_id); - }); - // If there are no more listening clients, terminate the streamer. - if streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { - streamers.remove(streamer_id); + pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { + let mut clients = self.clients.lock().unwrap(); + if let Some(listening_to) = clients.get_mut(&client_id) { + listening_to.remove(streamer_id); + + let mut streamers = self.streamers.write().unwrap(); + streamers + .get_mut(streamer_id) + .ok_or(StreamingManagerError::StreamerNotFound)? + .clients + .remove(&client_id); + + // If there are no more listening clients, terminate the streamer. + if streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { + streamers.remove(streamer_id); + } + Ok(()) + } else { + Err(StreamingManagerError::UnknownClient) } } - /// Broadcasts some event directly to listening clients. + /// Broadcasts some event to clients listening to it. /// /// In contrast to `StreamingManager::send`, which sends some data to a streamer, /// this method broadcasts an event to the listening *clients* directly, independently @@ -162,33 +192,35 @@ impl StreamingManager { pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { let mut clients = self.clients.lock().unwrap(); if clients.contains_key(&client_id) { - return Err(StreamingManagerError::ClientExists(client_id)); + return Err(StreamingManagerError::ClientExists); } clients.insert(client_id, HashSet::new()); - // Note that events queued in the channel are `Arc` shared. - // So 1024 long buffer won't be heavy on memory. + // Note that events queued in the channel are `Arc<` shared. + // So a 1024 long buffer isn't actually heavy on memory. Ok(self.controller.write().unwrap().create_channel(client_id, 1024)) } pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { - let streamer_ids = self - .clients - .lock() - .unwrap() - .remove(&client_id) - .ok_or(StreamingManagerError::UnknownClient(client_id))?; - self.controller.write().unwrap().remove_channel(&client_id); - + let mut clients = self.clients.lock().unwrap(); + // Remove the client from our known-clients map. + let listening_to = clients.remove(&client_id).ok_or(StreamingManagerError::UnknownClient)?; + // Remove the client from all the streamers it was listening to. let mut streamers = self.streamers.write().unwrap(); - for streamer_id in streamer_ids { + for streamer_id in listening_to { if let Some(streamer_info) = streamers.get_mut(&streamer_id) { streamer_info.clients.remove(&client_id); - }; + } else { + log::error!( + "Client {client_id} was listening to a non-existent streamer {streamer_id}. This is a bug!" + ); + } // If there are no more listening clients, terminate the streamer. if streamers.get(&streamer_id).map(|info| info.clients.len()) == Some(0) { streamers.remove(&streamer_id); } } + // Remove our channel with this client. + self.controller.write().unwrap().remove_channel(&client_id); Ok(()) } } From fb66bf242f277577f7a637bc2ed99bbe0725385e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 15:07:45 +0200 Subject: [PATCH 045/128] fix lint issues --- mm2src/mm2_event_stream/src/controller.rs | 1 - mm2src/mm2_event_stream/src/event.rs | 2 -- mm2src/mm2_event_stream/src/streamer.rs | 1 - 3 files changed, 4 deletions(-) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 6a9ee35a34..35fd54e0d3 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -1,4 +1,3 @@ -use parking_lot::Mutex; use std::{collections::{HashMap, HashSet}, sync::Arc}; use tokio::sync::mpsc::{self, Receiver, Sender}; diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index e45852f9b4..a69de4b9bd 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -1,8 +1,6 @@ use std::collections::HashSet; -use std::fmt; use std::sync::Arc; -use serde::Deserialize; use serde_json::Value as Json; #[derive(Default)] diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 499738cab0..1a6f2aaad4 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,7 +1,6 @@ use std::any::{self, Any}; use crate::manager::StreamingManager; -use crate::{Controller, Event}; use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; From 304c1799fef7c6824cc851915c7604dc6f173073 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 15:49:58 +0200 Subject: [PATCH 046/128] adjust only network & core crates to the new changes --- mm2src/mm2_core/src/data_asker.rs | 3 +- mm2src/mm2_core/src/mm_ctx.rs | 2 +- mm2src/mm2_event_stream/src/lib.rs | 1 + mm2src/mm2_event_stream/src/streamer.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 8 ++--- mm2src/mm2_net/src/sse_handler.rs | 42 +++++++++++++++++-------- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index 26c6016cec..e4fde93cf3 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -69,8 +69,7 @@ impl MmCtx { }); self.event_stream_manager - .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input, None)) - .await; + .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input, None)); match receiver.await { Ok(response) => match serde_json::from_value::(response) { diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 728337809d..9aeef80721 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{manager::StreamingManager, EventStreamConfiguration}; +use mm2_event_stream::{EventStreamConfiguration, StreamingManager}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 9c7a17dc1f..b2d7b0f4f6 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -8,4 +8,5 @@ pub mod streamer; pub use configuration::EventStreamConfiguration; pub use controller::Controller; pub use event::{Event, Filter}; +pub use manager::{StreamingManager, StreamingManagerError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 1a6f2aaad4..9a64e2f01a 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,6 +1,6 @@ use std::any::{self, Any}; -use crate::manager::StreamingManager; +use crate::StreamingManager; use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index ed96d3ef82..a38701a042 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; use serde_json::{json, Value as Json}; @@ -43,7 +43,7 @@ impl EventStreamer for NetworkEvent { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { @@ -70,9 +70,7 @@ impl EventStreamer for NetworkEvent { }); if previously_sent != event_data || self.config.always_send { - broadcaster - .broadcast(Event::new(self.streamer_id(), event_data.clone(), None)) - .await; + broadcaster.broadcast(Event::new(self.streamer_id(), event_data.clone(), None)); previously_sent = event_data; } diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index e126d6c560..f5f6579bc5 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -1,7 +1,7 @@ use hyper::{body::Bytes, Body, Request, Response}; use mm2_core::mm_ctx::MmArc; use serde_json::json; -use std::{collections::HashSet, convert::Infallible}; +use std::convert::Infallible; pub const SSE_ENDPOINT: &str = "/event-stream"; @@ -15,22 +15,38 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result(Bytes::from(format!("data: {data} \n\n"))); - } + // The event's filter will decide whether to expose the event data to this client or not. + // This happens based on the events that this client has subscribed to. + let (event_type, message) = event.get(); + let data = json!({ + "_type": event_type, + "message": message, + }); + + yield Ok::<_, hyper::Error>(Bytes::from(format!("data: {data} \n\n"))); } + // Inform the event stream manager that the client has disconnected. + event_stream_manager.remove_client(client_id).ok(); }); let response = Response::builder() From cb1e1c9712293ba22c1fe90041f3516c8fc94175 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 15:52:58 +0200 Subject: [PATCH 047/128] abandon the smart Filter object since events now pass through the streaming manager (who has all info regarding which clients are interested in what events) we don't need such a filter. also this filter won't work with the sub/pub arch anyways --- mm2src/mm2_core/src/data_asker.rs | 2 +- mm2src/mm2_event_stream/src/event.rs | 46 ++-------------------------- mm2src/mm2_event_stream/src/lib.rs | 2 +- mm2src/mm2_net/src/network_event.rs | 2 +- 4 files changed, 5 insertions(+), 47 deletions(-) diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index e4fde93cf3..8a9f78eb60 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -69,7 +69,7 @@ impl MmCtx { }); self.event_stream_manager - .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input, None)); + .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input)); match receiver.await { Ok(response) => match serde_json::from_value::(response) { diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index a69de4b9bd..f4acfdd09d 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -1,6 +1,3 @@ -use std::collections::HashSet; -use std::sync::Arc; - use serde_json::Value as Json; #[derive(Default)] @@ -10,12 +7,6 @@ pub struct Event { streamer_id: String, /// The message to be sent to the client. message: Json, - /// The filter object to be used to determine whether the event should be sent or not. - /// It could also alter the event content - /// - /// The filter is wrapped in an `Arc` since the event producer should use it as a singleton - /// by using the same filter over and over again with multiple events. - filter: Option>, /// Indicating whether this event is an error event or a normal one. error: bool, } @@ -23,22 +14,20 @@ pub struct Event { impl Event { /// Creates a new `Event` instance with the specified event type and message. #[inline] - pub fn new(streamer_id: String, message: Json, filter: Option>) -> Self { + pub fn new(streamer_id: String, message: Json) -> Self { Self { streamer_id, message, - filter, error: false, } } /// Create a new error `Event` instance with the specified error event type and message. #[inline] - pub fn err(streamer_id: String, message: Json, filter: Option>) -> Self { + pub fn err(streamer_id: String, message: Json) -> Self { Self { streamer_id, message, - filter, error: true, } } @@ -49,35 +38,4 @@ impl Event { let prefix = if self.error { "ERROR:" } else { "" }; (format!("{prefix}{}", self.streamer_id), self.message.clone()) } - - /// Returns the event type and message to be sent or `None` if the event should not be sent. - /// - /// Uses the `requested_events` to determine whether the event should be sent or not. - /// If `requested_events` is empty, this doesn't mean the event won't be sent, this is - /// decided by the event's filtering mechanism. - /// - /// `requested_events` could also be used to alter the event content (e.g. remove certain fields) - pub fn get_data(&self, requested_events: &HashSet) -> Option<(String, Json)> { - self.filter.as_ref().map_or_else( - // If no filter is set, send the event as is. - || Some((self.streamer_id.clone(), self.message.clone())), - |filter| { - filter - .filter(&self.message, requested_events) - .map(|message| (self.streamer_id.clone(), message)) - }, - ) - } -} - -/// A trait that defines the filtering mechanism for events. -/// -/// Each event has a filter that determines whether the event should be send out -/// to the client or not based on the client's requested events. -pub trait Filter: Send + Sync { - /// Filters the event based on the requested events. - /// - /// Returns the (maybe altered) message to be sent or `None` if the event should not be sent. - /// `requested_events` is a set of the events that the client asked to subscribe to (e.g. `BALANCE:BTC`) - fn filter(&self, message: &Json, requested_events: &HashSet) -> Option; } diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index b2d7b0f4f6..afe403dc68 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -7,6 +7,6 @@ pub mod streamer; // Re-export important types. pub use configuration::EventStreamConfiguration; pub use controller::Controller; -pub use event::{Event, Filter}; +pub use event::Event; pub use manager::{StreamingManager, StreamingManagerError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index a38701a042..7fad6b56a1 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -70,7 +70,7 @@ impl EventStreamer for NetworkEvent { }); if previously_sent != event_data || self.config.always_send { - broadcaster.broadcast(Event::new(self.streamer_id(), event_data.clone(), None)); + broadcaster.broadcast(Event::new(self.streamer_id(), event_data.clone())); previously_sent = event_data; } From 4d98d237621f742bf699454cc713ee6c9022c2f0 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 19 Jul 2024 16:11:07 +0200 Subject: [PATCH 048/128] rename Event::streamer_id -> Event::event_type they are the same thing, keeping the two names visible so their equivliance is clear --- mm2src/mm2_event_stream/src/event.rs | 11 ++++++----- mm2src/mm2_event_stream/src/streamer.rs | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index f4acfdd09d..f61818c383 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -4,7 +4,7 @@ use serde_json::Value as Json; /// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { /// The type of the event (balance, network, swap, etc...). - streamer_id: String, + event_type: String, /// The message to be sent to the client. message: Json, /// Indicating whether this event is an error event or a normal one. @@ -16,7 +16,7 @@ impl Event { #[inline] pub fn new(streamer_id: String, message: Json) -> Self { Self { - streamer_id, + event_type: streamer_id, message, error: false, } @@ -26,16 +26,17 @@ impl Event { #[inline] pub fn err(streamer_id: String, message: Json) -> Self { Self { - streamer_id, + event_type: streamer_id, message, error: true, } } - pub fn origin(&self) -> &str { &self.streamer_id } + /// Returns the `event_type` (the ID of the streamer firing this event). + pub fn origin(&self) -> &str { &self.event_type } pub fn get(&self) -> (String, Json) { let prefix = if self.error { "ERROR:" } else { "" }; - (format!("{prefix}{}", self.streamer_id), self.message.clone()) + (format!("{prefix}{}", self.event_type), self.message.clone()) } } diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 9a64e2f01a..d19f26220d 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -33,6 +33,8 @@ where /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. async fn handle( self, + // FIXME: Consider creating a new wrapper type around `StreamingManager` + // that won't expose any other method except `broadcast`. broadcaster: StreamingManager, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, From bc3266528171475efb3da01e8efa9856482757db Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 22 Jul 2024 12:37:56 +0200 Subject: [PATCH 049/128] adapt controller tests to the new arch --- mm2src/mm2_event_stream/src/controller.rs | 75 ++++++++--------------- mm2src/mm2_event_stream/src/lib.rs | 3 +- mm2src/mm2_event_stream/src/manager.rs | 2 +- 3 files changed, 29 insertions(+), 51 deletions(-) diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs index 35fd54e0d3..4de601fb9c 100644 --- a/mm2src/mm2_event_stream/src/controller.rs +++ b/mm2src/mm2_event_stream/src/controller.rs @@ -9,28 +9,18 @@ pub struct Controller { } impl Controller { - /// Creates a new channels controller - pub fn new() -> Self { - Self { - channels: HashMap::new(), - } - } - /// Creates a new channel and returns it's events receiver - pub fn create_channel(&mut self, channel_id: u64, concurrency: usize) -> Receiver> { + pub(crate) fn create_channel(&mut self, channel_id: u64, concurrency: usize) -> Receiver> { let (tx, rx) = mpsc::channel(concurrency); self.channels.insert(channel_id, tx); rx } /// Removes the channel from the controller - pub fn remove_channel(&mut self, channel_id: &u64) { self.channels.remove(channel_id); } - - /// Returns number of active channels - pub fn num_connections(&self) -> usize { self.channels.len() } + pub(crate) fn remove_channel(&mut self, channel_id: &u64) { self.channels.remove(channel_id); } /// Broadcast message to all channels - pub fn broadcast(&self, message: M, client_ids: Option<&HashSet>) { + pub(crate) fn broadcast(&self, message: M, client_ids: Option<&HashSet>) { let msg = Arc::new(message); for tx in self.channels(client_ids) { // Only `try_send` here. If the receiver's channel is full (receiver is slow), it will @@ -60,6 +50,7 @@ impl Controller { mod tests { use super::*; use common::cross_test; + use tokio::sync::mpsc::error::TryRecvError; common::cfg_wasm32! { use wasm_bindgen_test::*; @@ -67,24 +58,24 @@ mod tests { } cross_test!(test_create_channel_and_broadcast, { - let controller = Controller::new(); - let mut channel_receiver = controller.create_channel(1); + let mut controller = Controller::default(); + let mut channel_receiver = controller.create_channel(0, 1); - controller.broadcast("Message".to_string()).await; + controller.broadcast("Message".to_string(), None); let received_msg = channel_receiver.recv().await.unwrap(); assert_eq!(*received_msg, "Message".to_string()); }); cross_test!(test_multiple_channels_and_broadcast, { - let controller = Controller::new(); + let mut controller = Controller::default(); let mut receivers = Vec::new(); - for _ in 0..3 { - receivers.push(controller.create_channel(1)); + for channel_id in 0..3 { + receivers.push(controller.create_channel(channel_id, 1)); } - controller.broadcast("Message".to_string()).await; + controller.broadcast("Message".to_string(), None); for receiver in &mut receivers { let received_msg = receiver.recv().await.unwrap(); @@ -92,26 +83,15 @@ mod tests { } }); - cross_test!(test_channel_cleanup_on_drop, { - let controller: Controller<()> = Controller::new(); - let channel_receiver = controller.create_channel(1); - - assert_eq!(controller.num_connections(), 1); - - drop(channel_receiver); - - assert_eq!(controller.num_connections(), 0); - }); - cross_test!(test_broadcast_across_channels, { - let controller = Controller::new(); + let mut controller = Controller::default(); let mut receivers = Vec::new(); - for _ in 0..3 { - receivers.push(controller.create_channel(1)); + for channel_id in 0..3 { + receivers.push(controller.create_channel(channel_id, 1)); } - controller.broadcast("Message".to_string()).await; + controller.broadcast("Message".to_string(), None); for receiver in &mut receivers { let received_msg = receiver.recv().await.unwrap(); @@ -119,16 +99,18 @@ mod tests { } }); - cross_test!(test_multiple_messages_and_drop, { - let controller = Controller::new(); - let mut channel_receiver = controller.create_channel(6); + cross_test!(test_msgs_drop_past_capacity, { + let mut controller = Controller::default(); + let mut channel_receiver = controller.create_channel(0, 6); - controller.broadcast("Message 1".to_string()).await; - controller.broadcast("Message 2".to_string()).await; - controller.broadcast("Message 3".to_string()).await; - controller.broadcast("Message 4".to_string()).await; - controller.broadcast("Message 5".to_string()).await; - controller.broadcast("Message 6".to_string()).await; + controller.broadcast("Message 1".to_string(), None); + controller.broadcast("Message 2".to_string(), None); + controller.broadcast("Message 3".to_string(), None); + controller.broadcast("Message 4".to_string(), None); + controller.broadcast("Message 5".to_string(), None); + controller.broadcast("Message 6".to_string(), None); + // This message should be dropped. + controller.broadcast("Message 7".to_string(), None); let mut received_msgs = Vec::new(); for _ in 0..6 { @@ -143,9 +125,6 @@ mod tests { assert_eq!(*received_msgs[4], "Message 5".to_string()); assert_eq!(*received_msgs[5], "Message 6".to_string()); - // Drop the channel receiver to trigger the drop hook (remove itself from the controller). - drop(channel_receiver); - - assert_eq!(controller.num_connections(), 0); + assert!(matches!(channel_receiver.try_recv(), Err(TryRecvError::Empty))); }); } diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index afe403dc68..7a85762119 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,12 +1,11 @@ pub mod configuration; -pub mod controller; +mod controller; pub mod event; pub mod manager; pub mod streamer; // Re-export important types. pub use configuration::EventStreamConfiguration; -pub use controller::Controller; pub use event::Event; pub use manager::{StreamingManager, StreamingManagerError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 36591f80bc..eceb4b53f2 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,7 +2,7 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, RwLock}; -use crate::{Controller, Event, EventStreamer}; +use crate::{controller::Controller, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; From 2e95af16fbd92f3a756a4fcfc2c970268f0ba96e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 22 Jul 2024 13:33:58 +0200 Subject: [PATCH 050/128] fix compilation for the rest of mm2 this is still faulty though. we need to only add a streamer on client request and not out of the blue. ctrl+f `add(0,` --- mm2src/coins/eth.rs | 2 +- mm2src/coins/eth/eth_balance_events.rs | 14 ++--- .../tendermint/tendermint_balance_events.rs | 10 ++-- mm2src/coins/utxo/utxo_balance_events.rs | 58 ++----------------- .../utxo/utxo_builder/utxo_arc_builder.rs | 2 +- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_balance_streaming.rs | 10 ++-- .../src/tendermint_with_assets_activation.rs | 2 +- mm2src/mm2_main/src/heartbeat_event.rs | 8 +-- mm2src/mm2_main/src/lp_native_dex.rs | 4 +- 10 files changed, 29 insertions(+), 83 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index c1da8a2d92..18aec903a5 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5285,7 +5285,7 @@ impl EthCoin { let balance_streamer = EthBalanceEventStreamer::try_new(json!({}), self.clone()) .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))?; ctx.event_stream_manager - .add(balance_streamer, self.spawner()) + .add(0, balance_streamer, self.spawner()) .await .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {:?}", e)) } diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index f6c9d37df8..84cbc794db 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -4,7 +4,7 @@ use ethereum_types::Address; use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use instant::Instant; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_number::BigDecimal; use serde::Deserialize; use serde_json::Value as Json; @@ -152,13 +152,13 @@ impl EventStreamer for EthBalanceEventStreamer { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - async fn start_polling(streamer_id: String, broadcaster: Controller, coin: EthCoin, interval: f64) { + async fn start_polling(streamer_id: String, broadcaster: StreamingManager, coin: EthCoin, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, // our goal is to broadcast changed balances every x seconds. @@ -182,7 +182,7 @@ impl EventStreamer for EthBalanceEventStreamer { Err(e) => { log::error!("Failed getting addresses for {}. Error: {}", coin.ticker, e); let e = serde_json::to_value(e).expect("Serialization shouldn't fail."); - broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e)); sleep_remaining_time(interval, now).await; continue; }, @@ -215,15 +215,13 @@ impl EventStreamer for EthBalanceEventStreamer { ); let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); // FIXME: We should add the address in the error message. - broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e)); }, }; } if !balance_updates.is_empty() { - broadcaster - .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) - .await; + broadcaster.broadcast(Event::new(streamer_id.clone(), json!(balance_updates))); } sleep_remaining_time(interval, now).await; diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index 0a0b33db87..bcb516c5bf 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -4,7 +4,7 @@ use futures::channel::oneshot; use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; -use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; @@ -39,7 +39,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { @@ -139,7 +139,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { Err(e) => { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); - broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e)); continue; }, @@ -169,9 +169,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { } if !balance_updates.is_empty() { - broadcaster - .broadcast(Event::new(streamer_id.clone(), json!(balance_updates), None)) - .await; + broadcaster.broadcast(Event::new(streamer_id.clone(), json!(balance_updates))); } } } diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 01c53a26a1..37f11ce48f 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -3,10 +3,9 @@ use common::{executor::Timer, log, Future01CompatExt}; use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; -use mm2_event_stream::{Controller, Event, EventStreamer, Filter, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; -use std::sync::Arc; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; @@ -59,14 +58,13 @@ impl EventStreamer for UtxoBalanceEventStreamer { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); let coin = self.coin; - let filter = Arc::new(UtxoBalanceEventFilter::new(coin.ticker())); async fn subscribe_to_addresses( utxo: &UtxoCoinFields, @@ -124,13 +122,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { Err(e) => { log::error!("{e}"); - broadcaster - .broadcast(Event::err( - streamer_id.clone(), - json!({ "error": e }), - Some(filter.clone()), - )) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), json!({ "error": e }))); }, }; @@ -143,13 +135,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { Err(e) => { log::error!("{e}"); - broadcaster - .broadcast(Event::err( - streamer_id.clone(), - json!({ "error": e }), - Some(filter.clone()), - )) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), json!({ "error": e }))); }, }; @@ -199,9 +185,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { log::error!("Failed getting balance for '{ticker}'. Error: {e}"); let e = serde_json::to_value(e).expect("Serialization should't fail."); - broadcaster - .broadcast(Event::err(streamer_id.clone(), e, Some(filter.clone()))) - .await; + broadcaster.broadcast(Event::err(streamer_id.clone(), e)); continue; }, @@ -213,37 +197,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); - broadcaster - .broadcast(Event::new( - streamer_id.clone(), - json!(vec![payload]), - Some(filter.clone()), - )) - .await; + broadcaster.broadcast(Event::new(streamer_id.clone(), json!(vec![payload]))); } } } - -struct UtxoBalanceEventFilter { - /// The event name we are looking for to let this event pass. - event_name_match: String, -} - -impl UtxoBalanceEventFilter { - pub fn new(ticker: &str) -> Self { - Self { - // The client requested event must have our ticker in that format to pass through. - // fixme: Abandon this filter matching - event_name_match: String::new(), //format!("BALANCE_{}", UtxoBalanceEventStreamer::event_name(), ticker), - } - } -} - -impl Filter for UtxoBalanceEventFilter { - fn filter(&self, message: &Json, requested_events: &HashSet) -> Option { - if requested_events.is_empty() || requested_events.contains(&self.event_name_match) { - return Some(message.clone()); - } - None - } -} diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index d48efd17c9..e3193f8b1b 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -124,7 +124,7 @@ where })?; self.ctx .event_stream_manager - .add(utxo_streamer, utxo_arc.abortable_system.weak_spawner()) + .add(0, utxo_streamer, utxo_arc.abortable_system.weak_spawner()) .await .map_to_mm(|e| { UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to spawn utxo event streaming: {e:?}")) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 8d65e3c8da..795b46138e 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -664,7 +664,7 @@ impl ZCoin { let balance_streamer = ZCoinBalanceEventStreamer::try_new(json!({}), self.clone()) .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))?; ctx.event_stream_manager - .add(balance_streamer, self.spawner()) + .add(0, balance_streamer, self.spawner()) .await .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {:?}", e)) } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 6d7a6bac0d..2cf28be0b2 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -9,7 +9,7 @@ use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; -use mm2_event_stream::{Controller, Event, EventStreamer, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; use serde_json::Value as Json; use std::sync::Arc; @@ -42,7 +42,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput<()>, ) { @@ -63,15 +63,13 @@ impl EventStreamer for ZCoinBalanceEventStreamer { "balance": { "spendable": balance.spendable, "unspendable": balance.unspendable } }); - broadcaster - .broadcast(Event::new(streamer_id.clone(), payload, None)) - .await; + broadcaster.broadcast(Event::new(streamer_id.clone(), payload)); }, Err(err) => { let ticker = coin.ticker(); error!("Failed getting balance for '{ticker}'. Error: {err}"); let e = serde_json::to_value(err).expect("Serialization should't fail."); - return broadcaster.broadcast(Event::err(streamer_id.clone(), e, None)).await; + return broadcaster.broadcast(Event::err(streamer_id.clone(), e)); }, }; } diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index c8b0f887dd..cdd179189f 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -386,7 +386,7 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { }) .map_to_mm(|e| e)?; ctx.event_stream_manager - .add(balance_streamer, self.spawner()) + .add(0, balance_streamer, self.spawner()) .await .map_to_mm(|e| TendermintInitError { ticker: self.ticker().to_owned(), diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 3f27ced811..4244b02b98 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; -use mm2_event_stream::{Controller, Event, EventStreamer, NoDataIn, StreamHandlerInput}; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use serde::Deserialize; use serde_json::Value as Json; @@ -35,16 +35,14 @@ impl EventStreamer for HeartbeatEvent { async fn handle( self, - broadcaster: Controller, + broadcaster: StreamingManager, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { ready_tx.send(Ok(())).unwrap(); loop { - broadcaster - .broadcast(Event::new(self.streamer_id(), json!({}), None)) - .await; + broadcaster.broadcast(Event::new(self.streamer_id(), json!({}))); Timer::sleep(self.config.stream_interval_seconds).await; } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index e8aa9f8d3f..cbf5346414 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -429,7 +429,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { let network_steamer = NetworkEvent::try_new(config, ctx.clone()) .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))?; ctx.event_stream_manager - .add(network_steamer, ctx.spawner()) + .add(0, network_steamer, ctx.spawner()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e:?}")))?; } @@ -439,7 +439,7 @@ async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")) })?; ctx.event_stream_manager - .add(heartbeat_streamer, ctx.spawner()) + .add(0, heartbeat_streamer, ctx.spawner()) .await .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e:?}")))?; } From 7578ddfc0ecf28ae58b99e7dc8ee3e6d708e89c2 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 22 Jul 2024 14:33:11 +0200 Subject: [PATCH 051/128] fix wasm compilation --- mm2src/coins/utxo/utxo_tests.rs | 4 --- mm2src/mm2_event_stream/src/manager.rs | 4 +-- mm2src/mm2_net/src/wasm_event_stream.rs | 34 ++++++++++++------------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 868f558f1f..aec14d3dec 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -4030,8 +4030,6 @@ fn test_scan_for_new_addresses() { let client = NativeClient(Arc::new(NativeClientImpl::default())); let mut fields = utxo_coin_fields_for_test(UtxoRpcClientEnum::Native(client), None, false); - let ctx = MmCtxBuilder::new().into_mm_arc(); - fields.ctx = ctx.weak(); let mut hd_accounts = HDAccountsMap::new(); hd_accounts.insert(0, UtxoHDAccount { account_id: 0, @@ -4174,8 +4172,6 @@ fn test_get_new_address() { let client = NativeClient(Arc::new(NativeClientImpl::default())); let mut fields = utxo_coin_fields_for_test(UtxoRpcClientEnum::Native(client), None, false); - let ctx = MmCtxBuilder::new().into_mm_arc(); - fields.ctx = ctx.weak(); let mut hd_accounts = HDAccountsMap::new(); let hd_account_for_test = UtxoHDAccount { account_id: 0, diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index eceb4b53f2..acd5df89fb 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -16,7 +16,7 @@ pub enum StreamingManagerError { /// There is no streamer with the given ID. StreamerNotFound, /// Couldn't send the data to the streamer. - SendError, + SendError(String), /// The streamer doesn't accept an input. NoDataIn, /// Couldn't spawn the streamer. @@ -143,7 +143,7 @@ impl StreamingManager { let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; data_in .unbounded_send(Box::new(data)) - .map_err(|_| StreamingManagerError::SendError) + .map_err(|e| StreamingManagerError::SendError(e.to_string())) } /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. diff --git a/mm2src/mm2_net/src/wasm_event_stream.rs b/mm2src/mm2_net/src/wasm_event_stream.rs index 7dbcd33e95..ebb534189e 100644 --- a/mm2src/mm2_net/src/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/wasm_event_stream.rs @@ -1,6 +1,5 @@ use mm2_core::mm_ctx::MmArc; use serde_json::json; -use std::collections::HashSet; use web_sys::SharedWorker; struct SendableSharedWorker(SharedWorker); @@ -13,12 +12,8 @@ unsafe impl Send for SendableMessagePort {} /// Handles broadcasted messages from `mm2_event_stream` continuously for WASM. pub async fn handle_worker_stream(ctx: MmArc) { - let config = &ctx.event_stream_configuration; - - let channel_controller = ctx.event_stream_manager.controller(); - let mut rx = channel_controller.create_channel(config.total_active_events()); - - let worker_path = config + let worker_path = ctx + .event_stream_configuration .worker_path .to_str() .expect("worker_path contains invalid UTF-8 characters"); @@ -35,21 +30,24 @@ pub async fn handle_worker_stream(ctx: MmArc) { let port = SendableMessagePort(worker.0.port()); port.0.start(); - let filtered_events = HashSet::new(); - - while let Some(event) = rx.recv().await { - if let Some((event_type, message)) = event.get_data(&filtered_events) { - let data = json!({ - "_type": event_type, - "message": message, - }); - let message_js = wasm_bindgen::JsValue::from_str(&data.to_string()); + let event_stream_manager = ctx.event_stream_manager.clone(); + let mut rx = event_stream_manager + .new_client(0) + .expect("A different wasm client is already listening. Only one client is allowed at a time."); - port.0.post_message(&message_js) + while let Some(event) = rx.recv().await { + let (event_type, message) = event.get(); + let data = json!({ + "_type": event_type, + "message": message, + }); + let message_js = wasm_bindgen::JsValue::from_str(&data.to_string()); + port.0.post_message(&message_js) .expect("Failed to post a message to the SharedWorker.\n\ This could be due to the browser being incompatible.\n\ For more details, please refer to https://developer.mozilla.org/en-US/docs/Web/API/MessagePort/postMessage#browser_compatibility"); - } } + + event_stream_manager.remove_client(0).ok(); } From ee32fd1290fc66ffa0032c966887b182a47bb84f Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 22 Jul 2024 14:49:17 +0200 Subject: [PATCH 052/128] breaking: change mm2 event_stream_worker_path config parameter changed from: 'event_streaming_configuration' : { 'worker_path': 'path/to/worker.js', ..other_params } to 'event_stream_worker_path': 'path/to/worker.js' --- mm2src/coins/streaming_events_config.rs | 2 ++ mm2src/mm2_net/src/wasm_event_stream.rs | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs index 2d7eb93a44..56f197bc58 100644 --- a/mm2src/coins/streaming_events_config.rs +++ b/mm2src/coins/streaming_events_config.rs @@ -1,3 +1,5 @@ +// FIXME: Get rid of this. + use serde::Deserialize; use serde_json::Value as Json; use std::collections::HashMap; diff --git a/mm2src/mm2_net/src/wasm_event_stream.rs b/mm2src/mm2_net/src/wasm_event_stream.rs index ebb534189e..9bd2b90a98 100644 --- a/mm2src/mm2_net/src/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/wasm_event_stream.rs @@ -2,6 +2,8 @@ use mm2_core::mm_ctx::MmArc; use serde_json::json; use web_sys::SharedWorker; +const DEFAULT_WORKER_PATH: &str = "event_streaming_worker.js"; + struct SendableSharedWorker(SharedWorker); unsafe impl Send for SendableSharedWorker {} @@ -12,11 +14,9 @@ unsafe impl Send for SendableMessagePort {} /// Handles broadcasted messages from `mm2_event_stream` continuously for WASM. pub async fn handle_worker_stream(ctx: MmArc) { - let worker_path = ctx - .event_stream_configuration - .worker_path - .to_str() - .expect("worker_path contains invalid UTF-8 characters"); + let worker_path = ctx.conf["event_stream_worker_path"] + .as_str() + .unwrap_or(DEFAULT_WORKER_PATH); let worker = SendableSharedWorker( SharedWorker::new(worker_path).unwrap_or_else(|_| { panic!( From 35e9686dd2b8333afc3e0b55dea6dcb943e8d773 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 25 Jul 2024 13:23:34 +0200 Subject: [PATCH 053/128] bug: make sure the streamer is up before joining a new client along If a streamer is down (we wrote an event streamer that doesn't loop forever for some reason, or maybe the streamer paniced) we should start it agian the next time it is activated by any client, so the newly listening client gets the actual stream of events. Note that we don't keep older clients listening to that streamer, because: 1- the streamer writer probably wanted this effect, if the streamer is dead, clients should get no more events. 2- streamer initialization might fail, and if we keep the clients listening we will end up with `clients` & `streamers` maps out of sync. if we remove the clients if the streamer fails (and only if it fails), this will be a highly inconsistent side-effect. We should make sure streamers never exit (unless this is the wanted logic). For other cases like when the spawner used by the streamer is aborted (e.g. utxo balance streamer running on utxo coin spawner and then we disable the coin), we should make sure to use the approbriate spawner to exit only when the streamer's dependecy is no longer present. We should expect clients to re-enable the streamer is such cases (e.g. [enable utxo coin, enable blanace streamer, disable utxo coin, enable utxo coin] scenario doesn't re-enable the balance streamer by itself, the client should do [enable balance streamer] one more time) --- mm2src/mm2_event_stream/src/manager.rs | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index acd5df89fb..9ff92c89e2 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -35,7 +35,7 @@ struct StreamerInfo { /// Clients the streamer is serving for. clients: HashSet, /// The shutdown handle of the streamer. - _shutdown: oneshot::Sender<()>, + shutdown: oneshot::Sender<()>, } impl StreamerInfo { @@ -43,9 +43,11 @@ impl StreamerInfo { Self { data_in, clients: HashSet::new(), - _shutdown: shutdown, + shutdown, } } + + fn is_down(&self) -> bool { self.shutdown.is_canceled() } } #[derive(Clone, Default)] @@ -72,10 +74,14 @@ impl StreamingManager { spawner: WeakSpawner, ) -> Result<(), StreamingManagerError> { let streamer_id = streamer.streamer_id(); + // Remove the streamer if it died for some reason. + self.remove_streamer_if_down(&streamer_id); // Pre-checks before spawning the streamer. Inside another scope to drop the locks early. { let mut clients = self.clients.lock().unwrap(); + let mut streamers = self.streamers.write().unwrap(); + match clients.get(&client_id) { // We don't know that client. We don't have a connection to it. None => return Err(StreamingManagerError::UnknownClient), @@ -85,8 +91,9 @@ impl StreamingManager { }, _ => (), } + // If a streamer is already up and running, we won't spawn another one. - if let Some(streamer_info) = self.streamers.write().unwrap().get_mut(&streamer_id) { + if let Some(streamer_info) = streamers.get_mut(&streamer_id) { // Register the client as a listener to the streamer. streamer_info.clients.insert(client_id); // Register the streamer as listened-to by the client. @@ -223,4 +230,26 @@ impl StreamingManager { self.controller.write().unwrap().remove_channel(&client_id); Ok(()) } + + /// Removes a streamer if it is no longer running. + /// + /// Aside from us shutting down a streamer when all its clients are disconnected, + /// the streamer might die by itself (e.g. the spawner it was spawned with aborted). + /// In this case, we need to remove the streamer and de-list it from all clients. + fn remove_streamer_if_down(&self, streamer_id: &str) { + let mut clients = self.clients.lock().unwrap(); + let mut streamers = self.streamers.write().unwrap(); + if let Some(streamer_info) = streamers.get(streamer_id) { + // Remove the streamer from all clients listening to it. + if streamer_info.is_down() { + for client_id in &streamer_info.clients { + clients + .get_mut(client_id) + .map(|listening_to| listening_to.remove(streamer_id)); + } + // And remove the streamer from our registry. + streamers.remove(streamer_id); + } + } + } } From 58e5c723ff929856701d50c517d0bfb95e6d7ddd Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 22 Jul 2024 17:26:03 +0200 Subject: [PATCH 054/128] properly timeout in data asker `ask_for_data` didn't use the timeout that was passed over. the only way for it to timeout was for a client to reply for the asked data *after* the timeout --- mm2src/mm2_core/src/data_asker.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index 8a9f78eb60..21c5211047 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -1,3 +1,4 @@ +use common::custom_futures::timeout::FutureTimerExt; use common::expirable_map::ExpirableMap; use common::{HttpStatusCode, StatusCode}; use derive_more::Display; @@ -26,11 +27,12 @@ pub struct DataAsker { #[derive(Debug, Display)] pub enum AskForDataError { #[display( - fmt = "Expected JSON data, but given(from data provider) one was not deserializable: {:?}", + fmt = "Expected JSON data, but the received data (from data provider) was not deserializable: {:?}", _0 )] DeserializationError(serde_json::Error), Internal(String), + Timeout, } impl MmCtx { @@ -68,17 +70,19 @@ impl MmCtx { "data": data }); + // FIXME: Use broadcast_all here? self.event_stream_manager .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input)); - match receiver.await { - Ok(response) => match serde_json::from_value::(response) { + match receiver.timeout(timeout).await { + Ok(Ok(response)) => match serde_json::from_value::(response) { Ok(value) => Ok(value), Err(error) => MmError::err(AskForDataError::DeserializationError(error)), }, - Err(error) => MmError::err(AskForDataError::Internal(format!( - "Sender channel is not alive. {error}" + Ok(Err(error)) => MmError::err(AskForDataError::Internal(format!( + "Receiver channel is not alive. {error}" ))), + Err(_) => MmError::err(AskForDataError::Timeout), } } } From e3a411392d80bb1565939ff4fd5d23400f8cfa8c Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 24 Jul 2024 11:23:23 +0200 Subject: [PATCH 055/128] move balance network and other streamers to api enabling/disabling enabling and disabling streamers now take place in mm2_main/rpc/streaming_activations for modularity, we shouldn't enable streamers elsewhere so not to forget about them --- mm2src/coins/eth.rs | 18 +-- mm2src/coins/eth/eth_balance_events.rs | 31 ++-- mm2src/coins/eth/v2_activation.rs | 7 +- mm2src/coins/streaming_events_config.rs | 1 + .../tendermint/tendermint_balance_events.rs | 14 +- mm2src/coins/utxo.rs | 7 +- mm2src/coins/utxo/bch.rs | 4 + mm2src/coins/utxo/rpc_clients.rs | 1 + mm2src/coins/utxo/utxo_balance_events.rs | 17 +-- .../utxo/utxo_builder/utxo_arc_builder.rs | 13 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 6 +- mm2src/coins/utxo/utxo_common_tests.rs | 1 - mm2src/coins/z_coin.rs | 21 +-- mm2src/coins/z_coin/z_balance_streaming.rs | 14 +- .../src/tendermint_with_assets_activation.rs | 29 +--- mm2src/mm2_event_stream/src/manager.rs | 8 +- mm2src/mm2_main/src/heartbeat_event.rs | 19 ++- mm2src/mm2_main/src/lp_native_dex.rs | 27 ---- mm2src/mm2_main/src/rpc.rs | 1 + .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 27 ++++ .../src/rpc/streaming_activations/balance.rs | 133 ++++++++++++++++++ .../rpc/streaming_activations/heartbeat.rs | 54 +++++++ .../src/rpc/streaming_activations/mod.rs | 31 ++++ .../src/rpc/streaming_activations/network.rs | 54 +++++++ mm2src/mm2_net/src/network_event.rs | 21 ++- 25 files changed, 372 insertions(+), 187 deletions(-) create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/balance.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/mod.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/network.rs diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 18aec903a5..d3bd972278 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -63,7 +63,7 @@ use crypto::privkey::key_pair_from_secret; use crypto::{Bip44Chain, CryptoCtx, CryptoCtxError, GlobalHDAccountArc, KeyPairPolicy}; use derive_more::Display; use enum_derives::EnumFromStringify; -use eth_balance_events::EthBalanceEventStreamer; + use ethabi::{Contract, Function, Token}; use ethcore_transaction::tx_builders::TxBuilderError; use ethcore_transaction::{Action, TransactionWrapper, TransactionWrapperBuilder as UnSignedEthTxBuilder, @@ -134,7 +134,7 @@ cfg_native! { use std::path::PathBuf; } -mod eth_balance_events; +pub mod eth_balance_events; mod eth_rpc; #[cfg(test)] mod eth_tests; #[cfg(target_arch = "wasm32")] mod eth_wasm_tests; @@ -5281,15 +5281,6 @@ impl EthCoin { Box::new(fut.boxed().compat()) } - async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - let balance_streamer = EthBalanceEventStreamer::try_new(json!({}), self.clone()) - .map_err(|e| ERRL!("Failed to initialize eth balance streaming: {}", e))?; - ctx.event_stream_manager - .add(0, balance_streamer, self.spawner()) - .await - .map_err(|e| ERRL!("Failed to spawn eth balance streaming: {:?}", e)) - } - /// Requests the nonce from all available nodes and returns the highest nonce available with the list of nodes that returned the highest nonce. /// Transactions will be sent using the nodes that returned the highest nonce. pub fn get_addr_nonce( @@ -6372,10 +6363,7 @@ pub async fn eth_coin_from_conf_and_request( abortable_system, }; - let coin = EthCoin(Arc::new(coin)); - coin.spawn_balance_stream_if_enabled(ctx).await?; - - Ok(coin) + Ok(EthCoin(Arc::new(coin))) } /// Displays the address in mixed-case checksum form diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 84cbc794db..b5508628a5 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -11,39 +11,38 @@ use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; use super::EthCoin; -use crate::streaming_events_config::BalanceEventConfig; use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, BalanceError, CoinWithDerivationMethod}; #[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct SingleEthCoinConfig { +#[serde(deny_unknown_fields, default)] +struct EthBalanceStreamingConfig { /// The time in seconds to wait before re-polling the balance and streaming. - #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, } -const fn default_stream_interval() -> f64 { 10. } +impl Default for EthBalanceStreamingConfig { + fn default() -> Self { + Self { + stream_interval_seconds: 10.0, + } + } +} pub struct EthBalanceEventStreamer { - /// Whether the event is enabled for this coin. - enabled: bool, /// The period in seconds between each balance check. interval: f64, coin: EthCoin, } impl EthBalanceEventStreamer { - pub fn try_new(config: Json, coin: EthCoin) -> serde_json::Result { - let config: BalanceEventConfig = serde_json::from_value(config)?; - let coin_config: Option = match config.find_coin(&coin.ticker) { - // Try to parse the coin config. - Some(c) => Some(serde_json::from_value(c)?), - None => None, - }; + pub fn try_new(config: Option, coin: EthCoin) -> serde_json::Result { + let config: EthBalanceStreamingConfig = config + .map(|c| serde_json::from_value(c)) + .unwrap_or(Ok(Default::default()))?; + Ok(Self { - enabled: coin_config.is_some(), - interval: coin_config.map(|c| c.stream_interval_seconds).unwrap_or(0.0), + interval: config.stream_interval_seconds, coin, }) } diff --git a/mm2src/coins/eth/v2_activation.rs b/mm2src/coins/eth/v2_activation.rs index e41921508d..1a9b8b794c 100644 --- a/mm2src/coins/eth/v2_activation.rs +++ b/mm2src/coins/eth/v2_activation.rs @@ -620,12 +620,7 @@ pub async fn eth_coin_from_conf_and_request_v2( abortable_system, }; - let coin = EthCoin(Arc::new(coin)); - coin.spawn_balance_stream_if_enabled(ctx) - .await - .map_err(EthActivationV2Error::FailedSpawningBalanceEvents)?; - - Ok(coin) + Ok(EthCoin(Arc::new(coin))) } /// Processes the given `priv_key_policy` and generates corresponding `KeyPair`. diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs index 56f197bc58..abecb1b607 100644 --- a/mm2src/coins/streaming_events_config.rs +++ b/mm2src/coins/streaming_events_config.rs @@ -1,3 +1,4 @@ +#![allow(dead_code)] // FIXME: Get rid of this. use serde::Deserialize; diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index bcb516c5bf..abddf26bfc 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -6,29 +6,17 @@ use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_number::BigDecimal; -use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; use super::TendermintCoin; -use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::{tendermint::TendermintCommons, utxo::utxo_common::big_decimal_from_sat_unsigned, MarketCoinOps}; pub struct TendermintBalanceEventStreamer { - /// Whether the event is enabled for this coin. - enabled: bool, coin: TendermintCoin, } impl TendermintBalanceEventStreamer { - pub fn try_new(config: Json, coin: TendermintCoin) -> serde_json::Result { - let config: BalanceEventConfig = serde_json::from_value(config)?; - let enabled = match config.find_coin(coin.ticker()) { - // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, - None => false, - }; - Ok(Self { enabled, coin }) - } + pub fn new(coin: TendermintCoin) -> Self { Self { coin } } } #[async_trait] diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 40e77dcf6b..902840a8bb 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -56,7 +56,7 @@ use common::{now_sec, now_sec_u32}; use crypto::{Bip32Error, DerivationPath, HDPathToCoin, Secp256k1ExtendedPublicKey, StandardHDPathError}; use derive_more::Display; #[cfg(not(target_arch = "wasm32"))] use dirs::home_dir; -use futures::channel::mpsc::{Receiver as AsyncReceiver, Sender as AsyncSender, UnboundedReceiver, UnboundedSender}; +use futures::channel::mpsc::{Receiver as AsyncReceiver, Sender as AsyncSender, UnboundedSender}; use futures::compat::Future01CompatExt; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures01::Future; @@ -148,7 +148,6 @@ pub enum ScripthashNotification { } pub type ScripthashNotificationSender = Option>; -type ScripthashNotificationHandler = Option>>>; #[cfg(windows)] #[cfg(not(target_arch = "wasm32"))] @@ -618,10 +617,6 @@ pub struct UtxoCoinFields { /// This abortable system is used to spawn coin's related futures that should be aborted on coin deactivation /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, - /// This is used for balance event streaming implementation for UTXOs. - /// If balance event streaming isn't enabled, this value will always be `None`; otherwise, - /// it will be used for receiving scripthash notifications to re-fetch balances. - scripthash_notification_handler: ScripthashNotificationHandler, } #[derive(Debug, Display)] diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 60a3ae126a..1a4b6fa492 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -83,6 +83,10 @@ pub struct BchCoin { slp_tokens_infos: Arc>>, } +impl From for UtxoArc { + fn from(coin: BchCoin) -> Self { coin.utxo_arc } +} + #[allow(clippy::large_enum_variant)] pub enum IsSlpUtxoError { Rpc(UtxoRpcError), diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index f5150e4a9a..f2d14395c8 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -2607,6 +2607,7 @@ async fn electrum_process_json( if let Some(sender) = scripthash_notification_sender { debug!("Sending scripthash message"); + // FIXME: Nobody is listening to this channel. Use the streaming manager. if let Err(e) = sender.unbounded_send(ScripthashNotification::Triggered(scripthash.to_string())) { error!("Failed sending scripthash message. {e}"); diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 37f11ce48f..916f7bc0ab 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -4,11 +4,9 @@ use futures::channel::oneshot; use futures_util::StreamExt; use keys::Address; use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; -use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; -use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::{utxo::{output_script, rpc_clients::electrum_script_hash, utxo_common::{address_balance, address_to_scripthash}, @@ -28,25 +26,16 @@ macro_rules! try_or_continue { } pub struct UtxoBalanceEventStreamer { - /// Whether the event is enabled for this coin. - enabled: bool, coin: UtxoStandardCoin, } impl UtxoBalanceEventStreamer { - pub fn try_new(config: Json, utxo_arc: UtxoArc) -> serde_json::Result { - let config: BalanceEventConfig = serde_json::from_value(config)?; - let enabled = match config.find_coin(&utxo_arc.conf.ticker) { - // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, - None => false, - }; - Ok(Self { - enabled, + pub fn new(utxo_arc: UtxoArc) -> Self { + Self { // We wrap the UtxoArc in a UtxoStandardCoin for easier method accessibility. // The UtxoArc might belong to a different coin type though. coin: UtxoStandardCoin::from(utxo_arc), - }) + } } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs index e3193f8b1b..30efe09d27 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs @@ -1,5 +1,5 @@ use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientImpl, UtxoJsonRpcClientInfo, UtxoRpcClientEnum}; -use crate::utxo::utxo_balance_events::UtxoBalanceEventStreamer; + use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::{UtxoCoinBuildError, UtxoCoinBuilder, UtxoCoinBuilderCommonOps, UtxoFieldsWithGlobalHDBuilder, UtxoFieldsWithHardwareWalletBuilder, @@ -119,17 +119,6 @@ where spawn_block_header_utxo_loop(self.ticker, &utxo_arc, sync_handle, spv_conf); } - let utxo_streamer = UtxoBalanceEventStreamer::try_new(json!({}), utxo_arc.clone()).map_to_mm(|e| { - UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to initialize utxo event streaming: {e}")) - })?; - self.ctx - .event_stream_manager - .add(0, utxo_streamer, utxo_arc.abortable_system.weak_spawner()) - .await - .map_to_mm(|e| { - UtxoCoinBuildError::FailedSpawningBalanceEvents(format!("Failed to spawn utxo event streaming: {e:?}")) - })?; - Ok(result_coin) } } diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 2578b4d13a..bac10fb2cf 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -265,7 +265,7 @@ where let my_script_pubkey = output_script(&my_address).map(|script| script.to_bytes())?; - let (scripthash_notification_sender, scripthash_notification_handler) = { + let (scripthash_notification_sender, _) = { let (sender, receiver) = futures::channel::mpsc::unbounded(); (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) }; @@ -304,7 +304,6 @@ where block_headers_status_notifier, block_headers_status_watcher, abortable_system, - scripthash_notification_handler, }; Ok(coin) @@ -350,7 +349,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { address_format, }; - let (scripthash_notification_sender, scripthash_notification_handler) = { + let (scripthash_notification_sender, _) = { let (sender, receiver) = futures::channel::mpsc::unbounded(); (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) }; @@ -389,7 +388,6 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { block_headers_status_notifier, block_headers_status_watcher, abortable_system, - scripthash_notification_handler, }; Ok(coin) } diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index dde8ddd43a..f68f2ff475 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -148,7 +148,6 @@ pub(super) fn utxo_coin_fields_for_test( block_headers_status_notifier: None, block_headers_status_watcher: None, abortable_system: AbortableQueue::default(), - scripthash_notification_handler: None, } } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 795b46138e..ad9bdd6134 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,5 +1,5 @@ pub mod storage; -mod z_balance_streaming; +pub mod z_balance_streaming; mod z_coin_errors; #[cfg(all(test, feature = "zhtlc-native-tests"))] mod z_coin_native_tests; @@ -25,7 +25,7 @@ use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxDa use crate::utxo::{UnsupportedAddr, UtxoFeeDetails}; use crate::z_coin::storage::{BlockDbImpl, WalletDbShared}; use crate::z_coin::z_balance_streaming::ZBalanceEventHandler; -use crate::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; + use crate::z_coin::z_tx_history::{fetch_tx_history_from_db, ZCoinTxHistoryItem}; use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DexFee, FeeApproxStage, FoundSwapTxSpend, HistorySyncState, MakerSwapTakerCoin, MarketCoinOps, MmCoin, MmCoinEnum, @@ -659,15 +659,6 @@ impl ZCoin { paging_options: request.paging_options, }) } - - async fn spawn_balance_stream_if_enabled(&self, ctx: &MmArc) -> Result<(), String> { - let balance_streamer = ZCoinBalanceEventStreamer::try_new(json!({}), self.clone()) - .map_err(|e| ERRL!("Failed to initialize zcoin balance streaming: {}", e))?; - ctx.event_stream_manager - .add(0, balance_streamer, self.spawner()) - .await - .map_err(|e| ERRL!("Failed to spawn zcoin balance streaming: {:?}", e)) - } } impl AsRef for ZCoin { @@ -938,13 +929,7 @@ impl<'a> UtxoCoinBuilder for ZCoinBuilder<'a> { z_balance_event_handler, }); - let zcoin = ZCoin { utxo_arc, z_fields }; - zcoin - .spawn_balance_stream_if_enabled(self.ctx) - .await - .map_to_mm(ZCoinBuildError::FailedSpawningBalanceEvents)?; - - Ok(zcoin) + Ok(ZCoin { utxo_arc, z_fields }) } } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 2cf28be0b2..63e7561110 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -1,5 +1,4 @@ use crate::common::Future01CompatExt; -use crate::streaming_events_config::{BalanceEventConfig, EmptySubConfig}; use crate::z_coin::ZCoin; use crate::MarketCoinOps; @@ -10,28 +9,17 @@ use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; -use serde_json::Value as Json; use std::sync::Arc; pub type ZBalanceEventSender = UnboundedSender<()>; pub type ZBalanceEventHandler = Arc>>; pub struct ZCoinBalanceEventStreamer { - /// Whether the event is enabled for this coin. - enabled: bool, coin: ZCoin, } impl ZCoinBalanceEventStreamer { - pub fn try_new(config: Json, coin: ZCoin) -> serde_json::Result { - let config: BalanceEventConfig = serde_json::from_value(config)?; - let enabled = match config.find_coin(coin.ticker()) { - // This is just an extra check to make sure the config is correct (no config) - Some(c) => serde_json::from_value::(c).map(|_| true)?, - None => false, - }; - Ok(Self { enabled, coin }) - } + pub fn new(coin: ZCoin) -> Self { Self { coin } } } #[async_trait] diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index cdd179189f..4f12154dbf 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -10,7 +10,7 @@ use crate::prelude::*; use async_trait::async_trait; use coins::hd_wallet::HDPathAccountToAddressId; use coins::my_tx_history_v2::TxHistoryStorage; -use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; + use coins::tendermint::tendermint_tx_history_v2::tendermint_history_loop; use coins::tendermint::{tendermint_priv_key_policy, TendermintActivationPolicy, TendermintCoin, TendermintCommons, TendermintConf, TendermintInitError, TendermintInitErrorKind, TendermintProtocolInfo, @@ -25,7 +25,7 @@ use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::{json, Value as Json}; +use serde_json::{Value as Json}; use std::collections::{HashMap, HashSet}; impl TokenOf for TendermintToken { @@ -370,30 +370,9 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { async fn handle_balance_streaming( &self, - config: &EventStreamConfiguration, + _config: &EventStreamConfiguration, ) -> Result<(), MmError> { - let balance_streamer = - TendermintBalanceEventStreamer::try_new(json!({}), self.clone()).map_to_mm(|e| TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::BalanceStreamInitError(format!( - "Failed to initialize tendermint balance streaming: {e}" - )), - })?; - let ctx = MmArc::from_weak(&self.ctx) - .ok_or_else(|| TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::Internal("MM context must have been initialized already.".to_owned()), - }) - .map_to_mm(|e| e)?; - ctx.event_stream_manager - .add(0, balance_streamer, self.spawner()) - .await - .map_to_mm(|e| TendermintInitError { - ticker: self.ticker().to_owned(), - kind: TendermintInitErrorKind::BalanceStreamInitError(format!( - "Failed to spawn tendermint balance streaming handler: {e:?}" - )), - }) + Ok(()) } fn rpc_task_manager( diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 9ff92c89e2..277c44ef64 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -72,7 +72,7 @@ impl StreamingManager { client_id: u64, streamer: impl EventStreamer, spawner: WeakSpawner, - ) -> Result<(), StreamingManagerError> { + ) -> Result { let streamer_id = streamer.streamer_id(); // Remove the streamer if it died for some reason. self.remove_streamer_if_down(&streamer_id); @@ -99,7 +99,7 @@ impl StreamingManager { // Register the streamer as listened-to by the client. clients .get_mut(&client_id) - .map(|listening_to| listening_to.insert(streamer_id)); + .map(|listening_to| listening_to.insert(streamer_id.clone())); // FIXME: The streamer running is probably running with different configuration. // We might want to inform the client that the configuration they asked for wasn't // applied and return the active configuration instead? @@ -109,7 +109,7 @@ impl StreamingManager { // // Another restricted solution is to not let clients set the streamer configuration // and only allow it through MM2 configuration file. - return Ok(()); + return Ok(streamer_id); } } @@ -138,7 +138,7 @@ impl StreamingManager { // We no longer have a connection for it. return Err(StreamingManagerError::UnknownClient); } - Ok(()) + Ok(streamer_id) } /// Sends data to a streamer with `streamer_id`. diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 4244b02b98..efb99fd3ca 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -5,24 +5,31 @@ use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, Strea use serde::Deserialize; use serde_json::Value as Json; -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Deserialize)] +#[serde(deny_unknown_fields, default)] struct HeartbeatEventConfig { /// The time in seconds to wait before sending another ping event. - #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, } -const fn default_stream_interval() -> f64 { 5. } +impl Default for HeartbeatEventConfig { + fn default() -> Self { + Self { + stream_interval_seconds: 5.0, + } + } +} pub struct HeartbeatEvent { config: HeartbeatEventConfig, } impl HeartbeatEvent { - pub fn try_new(config: Json) -> Result { + pub fn try_new(config: Option) -> serde_json::Result { Ok(Self { - config: serde_json::from_value(config).map_err(|e| e.to_string())?, + config: config + .map(|c| serde_json::from_value(c)) + .unwrap_or(Ok(Default::default()))?, }) } } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index cbf5346414..e2c421ff5b 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -32,7 +32,6 @@ use mm2_libp2p::behaviours::atomicdex::{GossipsubConfig, DEPRECATED_NETID_LIST}; use mm2_libp2p::{spawn_gossipsub, AdexBehaviourError, NodeType, RelayAddress, RelayAddressError, SeedNodeInfo, SwarmRuntime, WssCerts}; use mm2_metrics::mm_gauge; -use mm2_net::network_event::NetworkEvent; use mm2_net::p2p::P2PContext; use rpc_task::RpcTaskError; use serde_json::{self as json}; @@ -45,7 +44,6 @@ use std::{fs, usize}; #[cfg(not(target_arch = "wasm32"))] use crate::mm2::database::init_and_migrate_sql_db; -use crate::mm2::heartbeat_event::HeartbeatEvent; use crate::mm2::lp_message_service::{init_message_service, InitMessageServiceError}; use crate::mm2::lp_network::{lp_network_ports, p2p_event_process_loop, NetIdError}; use crate::mm2::lp_ordermatch::{broadcast_maker_orders_keep_alive_loop, clean_memory_loop, init_ordermatch_context, @@ -424,29 +422,6 @@ fn migrate_db(ctx: &MmArc) -> MmInitResult<()> { #[cfg(not(target_arch = "wasm32"))] fn migration_1(_ctx: &MmArc) {} -async fn init_event_streaming(ctx: &MmArc) -> MmInitResult<()> { - if let Some(config) = ctx.event_stream_configuration.get_event("NETWORK") { - let network_steamer = NetworkEvent::try_new(config, ctx.clone()) - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to initialize network event: {e}")))?; - ctx.event_stream_manager - .add(0, network_steamer, ctx.spawner()) - .await - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn network event: {e:?}")))?; - } - - if let Some(config) = ctx.event_stream_configuration.get_event("HEARTBEAT") { - let heartbeat_streamer = HeartbeatEvent::try_new(config).map_to_mm(|e| { - MmInitError::EventStreamerInitFailed(format!("Failed to initialize heartbeat event: {e}")) - })?; - ctx.event_stream_manager - .add(0, heartbeat_streamer, ctx.spawner()) - .await - .map_to_mm(|e| MmInitError::EventStreamerInitFailed(format!("Failed to spawn heartbeat event: {e:?}")))?; - } - - Ok(()) -} - #[cfg(target_arch = "wasm32")] fn init_wasm_event_streaming(ctx: &MmArc) { if ctx.event_stream_configuration.total_active_events() != 0 { @@ -487,8 +462,6 @@ pub async fn lp_init_continue(ctx: MmArc) -> MmInitResult<()> { // an order and start new swap that might get started 2 times because of kick-start kick_start(ctx.clone()).await?; - init_event_streaming(&ctx).await?; - ctx.spawner().spawn(lp_ordermatch_loop(ctx.clone())); ctx.spawner().spawn(broadcast_maker_orders_keep_alive_loop(ctx.clone())); diff --git a/mm2src/mm2_main/src/rpc.rs b/mm2src/mm2_main/src/rpc.rs index b99a9a9ec3..8fca9eebca 100644 --- a/mm2src/mm2_main/src/rpc.rs +++ b/mm2src/mm2_main/src/rpc.rs @@ -50,6 +50,7 @@ mod dispatcher_legacy; #[path = "rpc/lp_commands/lp_commands_legacy.rs"] pub mod lp_commands_legacy; #[path = "rpc/rate_limiter.rs"] mod rate_limiter; +#[path = "rpc/streaming_activations/mod.rs"] mod streaming_activations; /// Lists the RPC method not requiring the "userpass" authentication. /// None is also public to skip auth and display proper error in case of method is missing diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 271fbf48cc..a7c4551625 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -1,3 +1,4 @@ +use super::streaming_activations; use super::{DispatcherError, DispatcherResult, PUBLIC_METHODS}; use crate::mm2::lp_native_dex::init_hw::{cancel_init_trezor, init_trezor, init_trezor_status, init_trezor_user_action}; #[cfg(target_arch = "wasm32")] @@ -145,6 +146,10 @@ async fn auth(request: &MmRpcRequest, ctx: &MmArc, client: &SocketAddr) -> Dispa } async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult>> { + if let Some(streaming_request) = request.method.strip_prefix("stream::") { + let streaming_request = streaming_request.to_string(); + return rpc_streaming_dispatcher(request, ctx, streaming_request).await; + } if let Some(task_method) = request.method.strip_prefix("task::") { let task_method = task_method.to_string(); return rpc_task_dispatcher(request, ctx, task_method).await; @@ -319,6 +324,28 @@ async fn rpc_task_dispatcher( } } +async fn rpc_streaming_dispatcher( + request: MmRpcRequest, + ctx: MmArc, + streaming_request: String, +) -> DispatcherResult>> { + match streaming_request.as_str() { + "balance::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_balance_streaming).await, + "balance::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_balance_streaming).await, + "network::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_network_streaming).await, + "network::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_network_streaming).await, + "heartbeat::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_heartbeat_streaming).await, + "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat_streaming).await, + // "eth_fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_eth_fee_estimation_streaming).await, + // "eth_fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_eth_fee_estimation_streaming).await, + // "swap_status::enable" => handle_mmrpc(ctx, request, enable_swap_status_streaming).await, + // "swap_status::disable" => handle_mmrpc(ctx, request, disable_swap_status_streaming).await, + // "order_status::enable" => handle_mmrpc(ctx, request, enable_order_status_streaming).await, + // "order_status::disable" => handle_mmrpc(ctx, request, disable_order_status_streaming).await, + _ => MmError::err(DispatcherError::NoSuchMethod), + } +} + /// `gui_storage` dispatcher. /// /// # Note diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs new file mode 100644 index 0000000000..959a7a6971 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -0,0 +1,133 @@ +//! RPC activation and deactivation for different balance event streamers. +use super::{DisableStreamingRequest, EnableStreamingResponse}; + +use coins::eth::eth_balance_events::EthBalanceEventStreamer; +use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; +use coins::utxo::utxo_balance_events::UtxoBalanceEventStreamer; +use coins::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; +use coins::{lp_coinfind, MmCoin, MmCoinEnum}; +use common::HttpStatusCode; +use http::StatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use serde_json::Value as Json; + +#[derive(Deserialize)] +pub struct EnableBalanceStreamingRequest { + pub client_id: u64, + pub coin: String, + pub config: Option, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum BalanceStreamingRequestError { + EnableError(String), + DisableError(String), + CoinNotFound, + CoinNotSupported, + Internal(String), +} + +impl HttpStatusCode for BalanceStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + BalanceStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + BalanceStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + BalanceStreamingRequestError::CoinNotFound => StatusCode::NOT_FOUND, + BalanceStreamingRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, + BalanceStreamingRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn enable_balance_streaming( + ctx: MmArc, + req: EnableBalanceStreamingRequest, +) -> MmResult { + let coin = lp_coinfind(&ctx, &req.coin) + .await + .map_err(BalanceStreamingRequestError::Internal)? + .ok_or(BalanceStreamingRequestError::CoinNotFound)?; + + let enable_result = match coin { + MmCoinEnum::UtxoCoin(coin) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::Bch(coin) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::QtumCoin(coin) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::EthCoin(coin) => { + let streamer = EthBalanceEventStreamer::try_new(req.config, coin.clone()) + .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}")))?; + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::ZCoin(coin) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + let streamer = ZCoinBalanceEventStreamer::new(coin.clone()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::Tendermint(coin) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + let streamer = TendermintBalanceEventStreamer::new(coin.clone()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + // FIXME: What about tokens?! + _ => Err(BalanceStreamingRequestError::CoinNotSupported)?, + }; + + enable_result + .map(EnableStreamingResponse::new) + .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}"))) +} + +pub async fn disable_balance_streaming( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult<(), BalanceStreamingRequestError> { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| BalanceStreamingRequestError::DisableError(format!("{e:?}"))) +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs new file mode 100644 index 0000000000..bbc63ceff9 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -0,0 +1,54 @@ +//! RPC activation and deactivation for the heartbeats. +use super::{DisableStreamingRequest, EnableStreamingResponse}; + +use crate::mm2::heartbeat_event::HeartbeatEvent; +use common::HttpStatusCode; +use http::StatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use serde_json::Value as Json; + +#[derive(Deserialize)] +pub struct EnableHeartbeatRequest { + pub client_id: u64, + pub config: Option, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum HeartbeatRequestError { + EnableError(String), + DisableError(String), +} + +impl HttpStatusCode for HeartbeatRequestError { + fn status_code(&self) -> StatusCode { + match self { + HeartbeatRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + HeartbeatRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + } + } +} + +pub async fn enable_heartbeat_streaming( + ctx: MmArc, + req: EnableHeartbeatRequest, +) -> MmResult { + let heartbeat_streamer = HeartbeatEvent::try_new(req.config) + .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}")))?; + ctx.event_stream_manager + .add(req.client_id, heartbeat_streamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}"))) +} + +pub async fn disable_heartbeat_streaming( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult<(), HeartbeatRequestError> { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| HeartbeatRequestError::DisableError(format!("{e:?}"))) +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs new file mode 100644 index 0000000000..ee94c17ef8 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -0,0 +1,31 @@ +mod balance; +mod network; +mod heartbeat; +mod eth_fee_estimator; + +// Re-exports +pub use balance::*; +pub use network::*; +pub use heartbeat::*; +pub use eth_fee_estimator::*; + +#[derive(Serialize)] +/// The success response for any event streaming activation endpoint. +pub struct EnableStreamingResponse { + pub streamer_id: String, +} + +impl EnableStreamingResponse { + fn new(streamer_id: String) -> Self { + Self { + streamer_id + } + } +} + +#[derive(Deserialize)] +/// The request used for any event streaming deactivation endpoint. +pub struct DisableStreamingRequest { + pub client_id: u64, + pub streamer_id: String, +} \ No newline at end of file diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs new file mode 100644 index 0000000000..546e1a4275 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -0,0 +1,54 @@ +//! RPC activation and deactivation for the network event streamer. +use super::{DisableStreamingRequest, EnableStreamingResponse}; + +use common::HttpStatusCode; +use http::StatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; +use mm2_net::network_event::NetworkEvent; + +use serde_json::Value as Json; + +#[derive(Deserialize)] +pub struct EnableNetworkStreamingRequest { + pub client_id: u64, + pub config: Option, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum NetworkStreamingRequestError { + EnableError(String), + DisableError(String), +} + +impl HttpStatusCode for NetworkStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + NetworkStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + NetworkStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + } + } +} + +pub async fn enable_network_streaming( + ctx: MmArc, + req: EnableNetworkStreamingRequest, +) -> MmResult { + let network_steamer = NetworkEvent::try_new(req.config, ctx.clone()) + .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}")))?; + ctx.event_stream_manager + .add(req.client_id, network_steamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}"))) +} + +pub async fn disable_network_streaming( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult<(), NetworkStreamingRequestError> { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| NetworkStreamingRequestError::DisableError(format!("{e:?}"))) +} diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 7fad6b56a1..c877c1ca28 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -8,18 +8,23 @@ use mm2_libp2p::behaviours::atomicdex; use serde::Deserialize; use serde_json::{json, Value as Json}; -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Deserialize)] +#[serde(deny_unknown_fields, default)] struct NetworkEventConfig { /// The time in seconds to wait after sending network info before sending another one. - #[serde(default = "default_stream_interval")] pub stream_interval_seconds: f64, /// Always (force) send network info data, even if it's the same as the previous one sent. - #[serde(default)] pub always_send: bool, } -const fn default_stream_interval() -> f64 { 5. } +impl Default for NetworkEventConfig { + fn default() -> Self { + Self { + stream_interval_seconds: 5.0, + always_send: false, + } + } +} pub struct NetworkEvent { config: NetworkEventConfig, @@ -27,9 +32,11 @@ pub struct NetworkEvent { } impl NetworkEvent { - pub fn try_new(config: Json, ctx: MmArc) -> Result { + pub fn try_new(config: Option, ctx: MmArc) -> serde_json::Result { Ok(Self { - config: serde_json::from_value(config).map_err(|e| e.to_string())?, + config: config + .map(|c| serde_json::from_value(c)) + .unwrap_or(Ok(Default::default()))?, ctx, }) } From 50978af656d3789cbb57c564e0dc95ffc10bfe45 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 25 Jul 2024 09:18:27 +0200 Subject: [PATCH 056/128] Add fee estimation to the streaming API I went with removing the fee estimation context and rpc methods but I fount that it is used by mm2 itself and not just for API usage. Maybe with some refactor we could get the mm2 usage decoupled from the API one, and abandon the fee context (it looks possible from the streaming API side). --- mm2src/coins/eth.rs | 65 +-- mm2src/coins/eth/eip1559_gas_fee.rs | 499 ------------------ mm2src/coins/eth/eth_balance_events.rs | 20 +- .../fee_estimation/eip1559/block_native.rs | 158 ++++++ .../eth/fee_estimation/eip1559/infura.rs | 127 +++++ .../coins/eth/fee_estimation/eip1559/mod.rs | 116 ++++ .../eth/fee_estimation/eip1559/simple.rs | 137 +++++ .../eth/fee_estimation/eth_fee_events.rs | 175 ++++++ mm2src/coins/eth/fee_estimation/mod.rs | 2 + .../coins/rpc_command/get_estimated_fees.rs | 7 +- mm2src/mm2_main/src/heartbeat_event.rs | 2 +- .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 11 +- .../streaming_activations/fee_estimation.rs | 72 +++ .../src/rpc/streaming_activations/mod.rs | 22 +- mm2src/mm2_net/src/network_event.rs | 9 +- 15 files changed, 845 insertions(+), 577 deletions(-) delete mode 100644 mm2src/coins/eth/eip1559_gas_fee.rs create mode 100644 mm2src/coins/eth/fee_estimation/eip1559/block_native.rs create mode 100644 mm2src/coins/eth/fee_estimation/eip1559/infura.rs create mode 100644 mm2src/coins/eth/fee_estimation/eip1559/mod.rs create mode 100644 mm2src/coins/eth/fee_estimation/eip1559/simple.rs create mode 100644 mm2src/coins/eth/fee_estimation/eth_fee_events.rs create mode 100644 mm2src/coins/eth/fee_estimation/mod.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index d3bd972278..83943b1c99 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -54,8 +54,8 @@ use async_trait::async_trait; use bitcrypto::{dhash160, keccak256, ripemd160, sha256}; use common::custom_futures::repeatable::{Ready, Retry, RetryOnError}; use common::custom_futures::timeout::FutureTimerExt; -use common::executor::{abortable_queue::AbortableQueue, AbortOnDropHandle, AbortSettings, AbortableSystem, - AbortedError, SpawnAbortable, Timer}; +use common::executor::{abortable_queue::AbortableQueue, AbortSettings, AbortableSystem, AbortedError, SpawnAbortable, + Timer}; use common::log::{debug, error, info, warn}; use common::number_type_casting::SafeTypeCastingNumbers; use common::{get_utc_timestamp, now_sec, small_rng, DEX_FEE_ADDR_RAW_PUBKEY}; @@ -155,10 +155,10 @@ use eth_withdraw::{EthWithdraw, InitEthWithdraw, StandardEthWithdraw}; mod nonce; use nonce::ParityNonce; -mod eip1559_gas_fee; -pub(crate) use eip1559_gas_fee::FeePerGasEstimated; -use eip1559_gas_fee::{BlocknativeGasApiCaller, FeePerGasSimpleEstimator, GasApiConfig, GasApiProvider, - InfuraGasApiCaller}; +pub mod fee_estimation; +use fee_estimation::eip1559::{block_native::BlocknativeGasApiCaller, infura::InfuraGasApiCaller, + simple::FeePerGasSimpleEstimator, FeeEstimatorState, FeePerGasEstimated, GasApiConfig, + GasApiProvider}; /// https://github.com/artemii235/etomic-swap/blob/master/contracts/EtomicSwap.sol /// Dev chain (195.201.137.5:8565) contract address: 0x83965C539899cC0F918552e5A26915de40ee8852 @@ -338,7 +338,8 @@ impl TryFrom for PayForGasOption { type GasDetails = (U256, PayForGasOption); -#[derive(Debug, Display, EnumFromStringify)] +#[derive(Debug, Display, EnumFromStringify, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] pub enum Web3RpcError { #[display(fmt = "Transport: {}", _0)] Transport(String), @@ -572,29 +573,6 @@ impl From for EthPrivKeyBuildPolicy { } } -/// Gas fee estimator loop context, runs a loop to estimate max fee and max priority fee per gas according to EIP-1559 for the next block -/// -/// This FeeEstimatorContext handles rpc requests which start and stop gas fee estimation loop and handles the loop itself. -/// FeeEstimatorContext keeps the latest estimated gas fees to return them on rpc request -pub(crate) struct FeeEstimatorContext { - /// Latest estimated gas fee values - pub(crate) estimated_fees: Arc>, - /// Handler for estimator loop graceful shutdown - pub(crate) abort_handler: AsyncMutex>, -} - -/// Gas fee estimator creation state -pub(crate) enum FeeEstimatorState { - /// Gas fee estimation not supported for this coin - CoinNotSupported, - /// Platform coin required to be enabled for gas fee estimation for this coin - PlatformCoinRequired, - /// Fee estimator created, use simple internal estimator - Simple(AsyncMutex), - /// Fee estimator created, use provider or simple internal estimator (if provider fails) - Provider(AsyncMutex), -} - /// pImpl idiom. pub struct EthCoinImpl { ticker: String, @@ -5081,20 +5059,25 @@ impl EthCoin { } /// Get gas base fee and suggest priority tip fees for the next block (see EIP-1559) - pub async fn get_eip1559_gas_fee(&self) -> Web3RpcResult { + pub async fn get_eip1559_gas_fee(&self, gas_api_conf: Option) -> Web3RpcResult { let coin = self.clone(); let history_estimator_fut = FeePerGasSimpleEstimator::estimate_fee_by_history(&coin); let ctx = MmArc::from_weak(&coin.ctx).ok_or_else(|| MmError::new(Web3RpcError::Internal("ctx is null".into())))?; - let gas_api_conf = ctx.conf["gas_api"].clone(); - if gas_api_conf.is_null() { - debug!("No eth gas api provider config, using only history estimator"); - return history_estimator_fut - .await - .map_err(|e| MmError::new(Web3RpcError::Internal(e.to_string()))); - } - let gas_api_conf: GasApiConfig = json::from_value(gas_api_conf) - .map_err(|e| MmError::new(Web3RpcError::InvalidGasApiConfig(e.to_string())))?; + // If a `gas_api_conf` was provided, use that. + let gas_api_conf = if let Some(gas_api_config) = gas_api_conf { + gas_api_config + } else { + let gas_api_conf = ctx.conf["gas_api"].clone(); + if gas_api_conf.is_null() { + debug!("No eth gas api provider config, using only history estimator"); + return history_estimator_fut + .await + .map_err(|e| MmError::new(Web3RpcError::Internal(e.to_string()))); + } + json::from_value(gas_api_conf) + .map_err(|e| MmError::new(Web3RpcError::InvalidGasApiConfig(e.to_string())))? + }; let provider_estimator_fut = match gas_api_conf.provider { GasApiProvider::Infura => InfuraGasApiCaller::fetch_infura_fee_estimation(&gas_api_conf.url).boxed(), GasApiProvider::Blocknative => { @@ -5125,7 +5108,7 @@ impl EthCoin { Ok(PayForGasOption::Legacy(LegacyGasPrice { gas_price })) }, SwapTxFeePolicy::Low | SwapTxFeePolicy::Medium | SwapTxFeePolicy::High => { - let fee_per_gas = coin.get_eip1559_gas_fee().await?; + let fee_per_gas = coin.get_eip1559_gas_fee(None).await?; let pay_result = match swap_fee_policy { SwapTxFeePolicy::Low => PayForGasOption::Eip1559(Eip1559FeePerGas { max_fee_per_gas: fee_per_gas.low.max_fee_per_gas, diff --git a/mm2src/coins/eth/eip1559_gas_fee.rs b/mm2src/coins/eth/eip1559_gas_fee.rs deleted file mode 100644 index 4d33781f39..0000000000 --- a/mm2src/coins/eth/eip1559_gas_fee.rs +++ /dev/null @@ -1,499 +0,0 @@ -//! Provides estimations of base and priority fee per gas or fetch estimations from a gas api provider - -use super::web3_transport::FeeHistoryResult; -use super::{Web3RpcError, Web3RpcResult}; -use crate::{wei_from_gwei_decimal, wei_to_gwei_decimal, EthCoin, NumConversError}; -use ethereum_types::U256; -use mm2_err_handle::mm_error::MmError; -use mm2_err_handle::or_mm_error::OrMmError; -use mm2_number::BigDecimal; -use num_traits::FromPrimitive; -use std::convert::TryFrom; -use url::Url; -use web3::types::BlockNumber; - -pub(crate) use gas_api::BlocknativeGasApiCaller; -pub(crate) use gas_api::InfuraGasApiCaller; - -use gas_api::{BlocknativeBlockPricesResponse, InfuraFeePerGas}; - -const FEE_PER_GAS_LEVELS: usize = 3; - -/// Indicates which provider was used to get fee per gas estimations -#[derive(Clone, Debug)] -pub enum EstimationSource { - /// filled by default values - Empty, - /// internal simple estimator - Simple, - Infura, - Blocknative, -} - -impl ToString for EstimationSource { - fn to_string(&self) -> String { - match self { - EstimationSource::Empty => "empty".into(), - EstimationSource::Simple => "simple".into(), - EstimationSource::Infura => "infura".into(), - EstimationSource::Blocknative => "blocknative".into(), - } - } -} - -impl Default for EstimationSource { - fn default() -> Self { Self::Empty } -} - -enum PriorityLevelId { - Low = 0, - Medium = 1, - High = 2, -} - -/// Supported gas api providers -#[derive(Deserialize)] -pub enum GasApiProvider { - Infura, - Blocknative, -} - -#[derive(Deserialize)] -pub struct GasApiConfig { - /// gas api provider name to use - pub provider: GasApiProvider, - /// gas api provider or proxy base url (scheme, host and port without the relative part) - pub url: Url, -} - -/// Priority level estimated max fee per gas -#[derive(Clone, Debug, Default)] -pub struct FeePerGasLevel { - /// estimated max priority tip fee per gas in wei - pub max_priority_fee_per_gas: U256, - /// estimated max fee per gas in wei - pub max_fee_per_gas: U256, - /// estimated transaction min wait time in mempool in ms for this priority level - pub min_wait_time: Option, - /// estimated transaction max wait time in mempool in ms for this priority level - pub max_wait_time: Option, -} - -/// Internal struct for estimated fee per gas for several priority levels, in wei -/// low/medium/high levels are supported -#[derive(Default, Debug, Clone)] -pub struct FeePerGasEstimated { - /// base fee for the next block in wei - pub base_fee: U256, - /// estimated low priority fee - pub low: FeePerGasLevel, - /// estimated medium priority fee - pub medium: FeePerGasLevel, - /// estimated high priority fee - pub high: FeePerGasLevel, - /// which estimator used - pub source: EstimationSource, - /// base trend (up or down) - pub base_fee_trend: String, - /// priority trend (up or down) - pub priority_fee_trend: String, -} - -impl TryFrom for FeePerGasEstimated { - type Error = MmError; - - fn try_from(infura_fees: InfuraFeePerGas) -> Result { - Ok(Self { - base_fee: wei_from_gwei_decimal!(&infura_fees.estimated_base_fee)?, - low: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.low.suggested_max_fee_per_gas)?, - max_priority_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.low.suggested_max_priority_fee_per_gas)?, - min_wait_time: Some(infura_fees.low.min_wait_time_estimate), - max_wait_time: Some(infura_fees.low.max_wait_time_estimate), - }, - medium: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.medium.suggested_max_fee_per_gas)?, - max_priority_fee_per_gas: wei_from_gwei_decimal!( - &infura_fees.medium.suggested_max_priority_fee_per_gas - )?, - min_wait_time: Some(infura_fees.medium.min_wait_time_estimate), - max_wait_time: Some(infura_fees.medium.max_wait_time_estimate), - }, - high: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.high.suggested_max_fee_per_gas)?, - max_priority_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.high.suggested_max_priority_fee_per_gas)?, - min_wait_time: Some(infura_fees.high.min_wait_time_estimate), - max_wait_time: Some(infura_fees.high.max_wait_time_estimate), - }, - source: EstimationSource::Infura, - base_fee_trend: infura_fees.base_fee_trend, - priority_fee_trend: infura_fees.priority_fee_trend, - }) - } -} - -impl TryFrom for FeePerGasEstimated { - type Error = MmError; - - fn try_from(block_prices: BlocknativeBlockPricesResponse) -> Result { - if block_prices.block_prices.is_empty() { - return Ok(FeePerGasEstimated::default()); - } - if block_prices.block_prices[0].estimated_prices.len() < FEE_PER_GAS_LEVELS { - return Ok(FeePerGasEstimated::default()); - } - Ok(Self { - base_fee: wei_from_gwei_decimal!(&block_prices.block_prices[0].base_fee_per_gas)?, - low: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[2].max_fee_per_gas - )?, - max_priority_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[2].max_priority_fee_per_gas - )?, - min_wait_time: None, - max_wait_time: None, - }, - medium: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[1].max_fee_per_gas - )?, - max_priority_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[1].max_priority_fee_per_gas - )?, - min_wait_time: None, - max_wait_time: None, - }, - high: FeePerGasLevel { - max_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[0].max_fee_per_gas - )?, - max_priority_fee_per_gas: wei_from_gwei_decimal!( - &block_prices.block_prices[0].estimated_prices[0].max_priority_fee_per_gas - )?, - min_wait_time: None, - max_wait_time: None, - }, - source: EstimationSource::Blocknative, - base_fee_trend: String::default(), - priority_fee_trend: String::default(), - }) - } -} - -/// Simple priority fee per gas estimator based on fee history -/// normally used if gas api provider is not available -pub(crate) struct FeePerGasSimpleEstimator {} - -impl FeePerGasSimpleEstimator { - // TODO: add minimal max fee and priority fee - /// depth to look for fee history to estimate priority fees - const FEE_PRIORITY_DEPTH: u64 = 5u64; - - /// percentiles to pass to eth_feeHistory - const HISTORY_PERCENTILES: [f64; FEE_PER_GAS_LEVELS] = [25.0, 50.0, 75.0]; - - /// percentile to predict next base fee over historical rewards - const BASE_FEE_PERCENTILE: f64 = 75.0; - - /// percentiles to calc max priority fee over historical rewards - const PRIORITY_FEE_PERCENTILES: [f64; FEE_PER_GAS_LEVELS] = [50.0, 50.0, 50.0]; - - /// adjustment for max fee per gas picked up by sampling - const ADJUST_MAX_FEE: [f64; FEE_PER_GAS_LEVELS] = [1.1, 1.175, 1.25]; // 1.25 assures max_fee_per_gas will be over next block base_fee - - /// adjustment for max priority fee picked up by sampling - const ADJUST_MAX_PRIORITY_FEE: [f64; FEE_PER_GAS_LEVELS] = [1.0, 1.0, 1.0]; - - /// block depth for eth_feeHistory - pub fn history_depth() -> u64 { Self::FEE_PRIORITY_DEPTH } - - /// percentiles for priority rewards obtained with eth_feeHistory - pub fn history_percentiles() -> &'static [f64] { &Self::HISTORY_PERCENTILES } - - /// percentile for vector - fn percentile_of(v: &[U256], percent: f64) -> U256 { - let mut v_mut = v.to_owned(); - v_mut.sort(); - - // validate bounds: - let percent = if percent > 100.0 { 100.0 } else { percent }; - let percent = if percent < 0.0 { 0.0 } else { percent }; - - let value_pos = ((v_mut.len() - 1) as f64 * percent / 100.0).round() as usize; - v_mut[value_pos] - } - - /// Estimate simplified gas priority fees based on fee history - pub async fn estimate_fee_by_history(coin: &EthCoin) -> Web3RpcResult { - let res: Result = coin - .eth_fee_history( - U256::from(Self::history_depth()), - BlockNumber::Latest, - Self::history_percentiles(), - ) - .await; - - match res { - Ok(fee_history) => Ok(Self::calculate_with_history(&fee_history)?), - Err(_) => MmError::err(Web3RpcError::Internal("Eth requests failed".into())), - } - } - - fn predict_base_fee(base_fees: &[U256]) -> U256 { Self::percentile_of(base_fees, Self::BASE_FEE_PERCENTILE) } - - fn priority_fee_for_level( - level: PriorityLevelId, - base_fee: BigDecimal, - fee_history: &FeeHistoryResult, - ) -> Web3RpcResult { - let level_index = level as usize; - let level_rewards = fee_history - .priority_rewards - .as_ref() - .or_mm_err(|| Web3RpcError::Internal("expected reward in eth_feeHistory".into()))? - .iter() - .map(|rewards| rewards.get(level_index).copied().unwrap_or_else(|| U256::from(0))) - .collect::>(); - - // Calculate the max priority fee per gas based on the rewards percentile. - let max_priority_fee_per_gas = Self::percentile_of(&level_rewards, Self::PRIORITY_FEE_PERCENTILES[level_index]); - // Convert the priority fee to BigDecimal gwei, falling back to 0 on error. - let max_priority_fee_per_gas_gwei = - wei_to_gwei_decimal!(max_priority_fee_per_gas).unwrap_or_else(|_| BigDecimal::from(0)); - - // Calculate the max fee per gas by adjusting the base fee and adding the priority fee. - let adjust_max_fee = - BigDecimal::from_f64(Self::ADJUST_MAX_FEE[level_index]).unwrap_or_else(|| BigDecimal::from(0)); - let adjust_max_priority_fee = - BigDecimal::from_f64(Self::ADJUST_MAX_PRIORITY_FEE[level_index]).unwrap_or_else(|| BigDecimal::from(0)); - - // TODO: consider use checked ops - let max_fee_per_gas_dec = base_fee * adjust_max_fee + max_priority_fee_per_gas_gwei * adjust_max_priority_fee; - - Ok(FeePerGasLevel { - max_priority_fee_per_gas, - max_fee_per_gas: wei_from_gwei_decimal!(&max_fee_per_gas_dec)?, - // TODO: Consider adding default wait times if applicable (and mark them as uncertain). - min_wait_time: None, - max_wait_time: None, - }) - } - - /// estimate priority fees by fee history - fn calculate_with_history(fee_history: &FeeHistoryResult) -> Web3RpcResult { - // For estimation of max fee and max priority fee we use latest block base_fee but adjusted. - // Apparently for this simple fee estimator for assured high priority we should assume - // that the real base_fee may go up by 1,25 (i.e. if the block is full). This is covered by high priority ADJUST_MAX_FEE multiplier - let latest_base_fee = fee_history - .base_fee_per_gas - .first() - .cloned() - .unwrap_or_else(|| U256::from(0)); - let latest_base_fee_dec = wei_to_gwei_decimal!(latest_base_fee).unwrap_or_else(|_| BigDecimal::from(0)); - - // The predicted base fee is not used for calculating eip1559 values here and is provided for other purposes - // (f.e if the caller would like to do own estimates of max fee and max priority fee) - let predicted_base_fee = Self::predict_base_fee(&fee_history.base_fee_per_gas); - Ok(FeePerGasEstimated { - base_fee: predicted_base_fee, - low: Self::priority_fee_for_level(PriorityLevelId::Low, latest_base_fee_dec.clone(), fee_history)?, - medium: Self::priority_fee_for_level(PriorityLevelId::Medium, latest_base_fee_dec.clone(), fee_history)?, - high: Self::priority_fee_for_level(PriorityLevelId::High, latest_base_fee_dec, fee_history)?, - source: EstimationSource::Simple, - base_fee_trend: String::default(), - priority_fee_trend: String::default(), - }) - } -} - -mod gas_api { - use std::convert::TryInto; - - use super::FeePerGasEstimated; - use crate::eth::{Web3RpcError, Web3RpcResult}; - use http::StatusCode; - use mm2_err_handle::mm_error::MmError; - use mm2_err_handle::prelude::*; - use mm2_net::transport::slurp_url_with_headers; - use mm2_number::BigDecimal; - use serde_json::{self as json}; - use url::Url; - - lazy_static! { - /// API key for testing - static ref INFURA_GAS_API_AUTH_TEST: String = std::env::var("INFURA_GAS_API_AUTH_TEST").unwrap_or_default(); - } - - #[derive(Clone, Debug, Deserialize)] - pub(crate) struct InfuraFeePerGasLevel { - #[serde(rename = "suggestedMaxPriorityFeePerGas")] - pub suggested_max_priority_fee_per_gas: BigDecimal, - #[serde(rename = "suggestedMaxFeePerGas")] - pub suggested_max_fee_per_gas: BigDecimal, - #[serde(rename = "minWaitTimeEstimate")] - pub min_wait_time_estimate: u32, - #[serde(rename = "maxWaitTimeEstimate")] - pub max_wait_time_estimate: u32, - } - - /// Infura gas api response - /// see https://docs.infura.io/api/infura-expansion-apis/gas-api/api-reference/gasprices-type2 - #[allow(dead_code)] - #[derive(Debug, Deserialize)] - pub(crate) struct InfuraFeePerGas { - pub low: InfuraFeePerGasLevel, - pub medium: InfuraFeePerGasLevel, - pub high: InfuraFeePerGasLevel, - #[serde(rename = "estimatedBaseFee")] - pub estimated_base_fee: BigDecimal, - #[serde(rename = "networkCongestion")] - pub network_congestion: BigDecimal, - #[serde(rename = "latestPriorityFeeRange")] - pub latest_priority_fee_range: Vec, - #[serde(rename = "historicalPriorityFeeRange")] - pub historical_priority_fee_range: Vec, - #[serde(rename = "historicalBaseFeeRange")] - pub historical_base_fee_range: Vec, - #[serde(rename = "priorityFeeTrend")] - pub priority_fee_trend: String, // we are not using enum here bcz values not mentioned in docs could be received - #[serde(rename = "baseFeeTrend")] - pub base_fee_trend: String, - } - - /// Infura gas api provider caller - #[allow(dead_code)] - pub(crate) struct InfuraGasApiCaller {} - - #[allow(dead_code)] - impl InfuraGasApiCaller { - const INFURA_GAS_FEES_ENDPOINT: &'static str = "networks/1/suggestedGasFees"; // Support only main chain - - fn get_infura_gas_api_url(base_url: &Url) -> (Url, Vec<(&'static str, &'static str)>) { - let mut url = base_url.clone(); - url.set_path(Self::INFURA_GAS_FEES_ENDPOINT); - let headers = vec![("Authorization", INFURA_GAS_API_AUTH_TEST.as_str())]; - (url, headers) - } - - async fn make_infura_gas_api_request( - url: &Url, - headers: Vec<(&'static str, &'static str)>, - ) -> Result> { - let resp = slurp_url_with_headers(url.as_str(), headers) - .await - .mm_err(|e| e.to_string())?; - if resp.0 != StatusCode::OK { - let error = format!("{} failed with status code {}", url, resp.0); - return MmError::err(error); - } - let estimated_fees = json::from_slice(&resp.2).map_to_mm(|e| e.to_string())?; - Ok(estimated_fees) - } - - /// Fetch fee per gas estimations from infura provider - pub async fn fetch_infura_fee_estimation(base_url: &Url) -> Web3RpcResult { - let (url, headers) = Self::get_infura_gas_api_url(base_url); - let infura_estimated_fees = Self::make_infura_gas_api_request(&url, headers) - .await - .mm_err(Web3RpcError::Transport)?; - infura_estimated_fees.try_into().mm_err(Into::into) - } - } - - lazy_static! { - /// API key for testing - static ref BLOCKNATIVE_GAS_API_AUTH_TEST: String = std::env::var("BLOCKNATIVE_GAS_API_AUTH_TEST").unwrap_or_default(); - } - - #[allow(dead_code)] - #[derive(Clone, Debug, Deserialize)] - pub(crate) struct BlocknativeBlockPrices { - #[serde(rename = "blockNumber")] - pub block_number: u32, - #[serde(rename = "estimatedTransactionCount")] - pub estimated_transaction_count: u32, - #[serde(rename = "baseFeePerGas")] - pub base_fee_per_gas: BigDecimal, - #[serde(rename = "estimatedPrices")] - pub estimated_prices: Vec, - } - - #[allow(dead_code)] - #[derive(Clone, Debug, Deserialize)] - pub(crate) struct BlocknativeEstimatedPrices { - pub confidence: u32, - pub price: BigDecimal, - #[serde(rename = "maxPriorityFeePerGas")] - pub max_priority_fee_per_gas: BigDecimal, - #[serde(rename = "maxFeePerGas")] - pub max_fee_per_gas: BigDecimal, - } - - /// Blocknative gas prices response - /// see https://docs.blocknative.com/gas-prediction/gas-platform - #[allow(dead_code)] - #[derive(Debug, Deserialize)] - pub(crate) struct BlocknativeBlockPricesResponse { - pub system: String, - pub network: String, - pub unit: String, - #[serde(rename = "maxPrice")] - pub max_price: BigDecimal, - #[serde(rename = "currentBlockNumber")] - pub current_block_number: u32, - #[serde(rename = "msSinceLastBlock")] - pub ms_since_last_block: u32, - #[serde(rename = "blockPrices")] - pub block_prices: Vec, - } - - /// Blocknative gas api provider caller - #[allow(dead_code)] - pub(crate) struct BlocknativeGasApiCaller {} - - #[allow(dead_code)] - impl BlocknativeGasApiCaller { - const BLOCKNATIVE_GAS_PRICES_ENDPOINT: &'static str = "gasprices/blockprices"; - const BLOCKNATIVE_GAS_PRICES_LOW: &'static str = "10"; - const BLOCKNATIVE_GAS_PRICES_MEDIUM: &'static str = "50"; - const BLOCKNATIVE_GAS_PRICES_HIGH: &'static str = "90"; - - fn get_blocknative_gas_api_url(base_url: &Url) -> (Url, Vec<(&'static str, &'static str)>) { - let mut url = base_url.clone(); - url.set_path(Self::BLOCKNATIVE_GAS_PRICES_ENDPOINT); - url.query_pairs_mut() - .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_LOW) - .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_MEDIUM) - .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_HIGH) - .append_pair("withBaseFees", "true"); - - let headers = vec![("Authorization", BLOCKNATIVE_GAS_API_AUTH_TEST.as_str())]; - (url, headers) - } - - async fn make_blocknative_gas_api_request( - url: &Url, - headers: Vec<(&'static str, &'static str)>, - ) -> Result> { - let resp = slurp_url_with_headers(url.as_str(), headers) - .await - .mm_err(|e| e.to_string())?; - if resp.0 != StatusCode::OK { - let error = format!("{} failed with status code {}", url, resp.0); - return MmError::err(error); - } - let block_prices = json::from_slice(&resp.2).map_err(|e| e.to_string())?; - Ok(block_prices) - } - - /// Fetch fee per gas estimations from blocknative provider - pub async fn fetch_blocknative_fee_estimation(base_url: &Url) -> Web3RpcResult { - let (url, headers) = Self::get_blocknative_gas_api_url(base_url); - let block_prices = Self::make_blocknative_gas_api_request(&url, headers) - .await - .mm_err(Web3RpcError::Transport)?; - block_prices.try_into().mm_err(Into::into) - } - } -} diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index b5508628a5..e173d6c471 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -1,19 +1,19 @@ -use async_trait::async_trait; +use super::EthCoin; +use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, + BalanceError, CoinWithDerivationMethod}; use common::{executor::Timer, log, Future01CompatExt}; -use ethereum_types::Address; -use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; -use instant::Instant; use mm2_err_handle::prelude::MmError; use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_number::BigDecimal; + +use async_trait::async_trait; +use ethereum_types::Address; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; +use instant::Instant; use serde::Deserialize; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; -use super::EthCoin; -use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, - BalanceError, CoinWithDerivationMethod}; - #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] struct EthBalanceStreamingConfig { @@ -37,9 +37,7 @@ pub struct EthBalanceEventStreamer { impl EthBalanceEventStreamer { pub fn try_new(config: Option, coin: EthCoin) -> serde_json::Result { - let config: EthBalanceStreamingConfig = config - .map(|c| serde_json::from_value(c)) - .unwrap_or(Ok(Default::default()))?; + let config: EthBalanceStreamingConfig = config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?; Ok(Self { interval: config.stream_interval_seconds, diff --git a/mm2src/coins/eth/fee_estimation/eip1559/block_native.rs b/mm2src/coins/eth/fee_estimation/eip1559/block_native.rs new file mode 100644 index 0000000000..2f301e7c85 --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/eip1559/block_native.rs @@ -0,0 +1,158 @@ +use super::{EstimationSource, FeePerGasEstimated, FeePerGasLevel, FEE_PER_GAS_LEVELS}; +use crate::eth::{Web3RpcError, Web3RpcResult}; +use crate::{wei_from_gwei_decimal, NumConversError}; +use mm2_err_handle::mm_error::MmError; +use mm2_err_handle::prelude::*; +use mm2_net::transport::slurp_url_with_headers; +use mm2_number::BigDecimal; + +use http::StatusCode; +use serde_json::{self as json}; +use std::convert::TryFrom; +use std::convert::TryInto; +use url::Url; + +lazy_static! { + /// API key for testing + static ref BLOCKNATIVE_GAS_API_AUTH_TEST: String = std::env::var("BLOCKNATIVE_GAS_API_AUTH_TEST").unwrap_or_default(); +} + +#[allow(dead_code)] +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct BlocknativeBlockPrices { + #[serde(rename = "blockNumber")] + pub block_number: u32, + #[serde(rename = "estimatedTransactionCount")] + pub estimated_transaction_count: u32, + #[serde(rename = "baseFeePerGas")] + pub base_fee_per_gas: BigDecimal, + #[serde(rename = "estimatedPrices")] + pub estimated_prices: Vec, +} + +#[allow(dead_code)] +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct BlocknativeEstimatedPrices { + pub confidence: u32, + pub price: BigDecimal, + #[serde(rename = "maxPriorityFeePerGas")] + pub max_priority_fee_per_gas: BigDecimal, + #[serde(rename = "maxFeePerGas")] + pub max_fee_per_gas: BigDecimal, +} + +/// Blocknative gas prices response +/// see https://docs.blocknative.com/gas-prediction/gas-platform +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub(crate) struct BlocknativeBlockPricesResponse { + pub system: String, + pub network: String, + pub unit: String, + #[serde(rename = "maxPrice")] + pub max_price: BigDecimal, + #[serde(rename = "currentBlockNumber")] + pub current_block_number: u32, + #[serde(rename = "msSinceLastBlock")] + pub ms_since_last_block: u32, + #[serde(rename = "blockPrices")] + pub block_prices: Vec, +} + +impl TryFrom for FeePerGasEstimated { + type Error = MmError; + + fn try_from(block_prices: BlocknativeBlockPricesResponse) -> Result { + if block_prices.block_prices.is_empty() { + return Ok(FeePerGasEstimated::default()); + } + if block_prices.block_prices[0].estimated_prices.len() < FEE_PER_GAS_LEVELS { + return Ok(FeePerGasEstimated::default()); + } + Ok(Self { + base_fee: wei_from_gwei_decimal!(&block_prices.block_prices[0].base_fee_per_gas)?, + low: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[2].max_fee_per_gas + )?, + max_priority_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[2].max_priority_fee_per_gas + )?, + min_wait_time: None, + max_wait_time: None, + }, + medium: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[1].max_fee_per_gas + )?, + max_priority_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[1].max_priority_fee_per_gas + )?, + min_wait_time: None, + max_wait_time: None, + }, + high: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[0].max_fee_per_gas + )?, + max_priority_fee_per_gas: wei_from_gwei_decimal!( + &block_prices.block_prices[0].estimated_prices[0].max_priority_fee_per_gas + )?, + min_wait_time: None, + max_wait_time: None, + }, + source: EstimationSource::Blocknative, + base_fee_trend: String::default(), + priority_fee_trend: String::default(), + }) + } +} + +/// Blocknative gas api provider caller +#[allow(dead_code)] +pub(crate) struct BlocknativeGasApiCaller {} + +#[allow(dead_code)] +impl BlocknativeGasApiCaller { + const BLOCKNATIVE_GAS_PRICES_ENDPOINT: &'static str = "gasprices/blockprices"; + const BLOCKNATIVE_GAS_PRICES_LOW: &'static str = "10"; + const BLOCKNATIVE_GAS_PRICES_MEDIUM: &'static str = "50"; + const BLOCKNATIVE_GAS_PRICES_HIGH: &'static str = "90"; + + fn get_blocknative_gas_api_url(base_url: &Url) -> (Url, Vec<(&'static str, &'static str)>) { + let mut url = base_url.clone(); + url.set_path(Self::BLOCKNATIVE_GAS_PRICES_ENDPOINT); + url.query_pairs_mut() + .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_LOW) + .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_MEDIUM) + .append_pair("confidenceLevels", Self::BLOCKNATIVE_GAS_PRICES_HIGH) + .append_pair("withBaseFees", "true"); + + let headers = vec![("Authorization", BLOCKNATIVE_GAS_API_AUTH_TEST.as_str())]; + (url, headers) + } + + async fn make_blocknative_gas_api_request( + url: &Url, + headers: Vec<(&'static str, &'static str)>, + ) -> Result> { + let resp = slurp_url_with_headers(url.as_str(), headers) + .await + .mm_err(|e| e.to_string())?; + if resp.0 != StatusCode::OK { + let error = format!("{} failed with status code {}", url, resp.0); + return MmError::err(error); + } + let block_prices = json::from_slice(&resp.2).map_err(|e| e.to_string())?; + Ok(block_prices) + } + + /// Fetch fee per gas estimations from blocknative provider + pub async fn fetch_blocknative_fee_estimation(base_url: &Url) -> Web3RpcResult { + let (url, headers) = Self::get_blocknative_gas_api_url(base_url); + let block_prices = Self::make_blocknative_gas_api_request(&url, headers) + .await + .mm_err(Web3RpcError::Transport)?; + block_prices.try_into().mm_err(Into::into) + } +} diff --git a/mm2src/coins/eth/fee_estimation/eip1559/infura.rs b/mm2src/coins/eth/fee_estimation/eip1559/infura.rs new file mode 100644 index 0000000000..b6aa4e84d7 --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/eip1559/infura.rs @@ -0,0 +1,127 @@ +use super::{EstimationSource, FeePerGasEstimated, FeePerGasLevel}; +use crate::eth::{Web3RpcError, Web3RpcResult}; +use crate::{wei_from_gwei_decimal, NumConversError}; +use mm2_err_handle::mm_error::MmError; +use mm2_err_handle::prelude::*; +use mm2_net::transport::slurp_url_with_headers; +use mm2_number::BigDecimal; + +use http::StatusCode; +use serde_json::{self as json}; +use std::convert::TryFrom; +use std::convert::TryInto; +use url::Url; + +lazy_static! { + /// API key for testing + static ref INFURA_GAS_API_AUTH_TEST: String = std::env::var("INFURA_GAS_API_AUTH_TEST").unwrap_or_default(); +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct InfuraFeePerGasLevel { + #[serde(rename = "suggestedMaxPriorityFeePerGas")] + pub suggested_max_priority_fee_per_gas: BigDecimal, + #[serde(rename = "suggestedMaxFeePerGas")] + pub suggested_max_fee_per_gas: BigDecimal, + #[serde(rename = "minWaitTimeEstimate")] + pub min_wait_time_estimate: u32, + #[serde(rename = "maxWaitTimeEstimate")] + pub max_wait_time_estimate: u32, +} + +/// Infura gas api response +/// see https://docs.infura.io/api/infura-expansion-apis/gas-api/api-reference/gasprices-type2 +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub(crate) struct InfuraFeePerGas { + pub low: InfuraFeePerGasLevel, + pub medium: InfuraFeePerGasLevel, + pub high: InfuraFeePerGasLevel, + #[serde(rename = "estimatedBaseFee")] + pub estimated_base_fee: BigDecimal, + #[serde(rename = "networkCongestion")] + pub network_congestion: BigDecimal, + #[serde(rename = "latestPriorityFeeRange")] + pub latest_priority_fee_range: Vec, + #[serde(rename = "historicalPriorityFeeRange")] + pub historical_priority_fee_range: Vec, + #[serde(rename = "historicalBaseFeeRange")] + pub historical_base_fee_range: Vec, + #[serde(rename = "priorityFeeTrend")] + pub priority_fee_trend: String, // we are not using enum here bcz values not mentioned in docs could be received + #[serde(rename = "baseFeeTrend")] + pub base_fee_trend: String, +} + +impl TryFrom for FeePerGasEstimated { + type Error = MmError; + + fn try_from(infura_fees: InfuraFeePerGas) -> Result { + Ok(Self { + base_fee: wei_from_gwei_decimal!(&infura_fees.estimated_base_fee)?, + low: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.low.suggested_max_fee_per_gas)?, + max_priority_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.low.suggested_max_priority_fee_per_gas)?, + min_wait_time: Some(infura_fees.low.min_wait_time_estimate), + max_wait_time: Some(infura_fees.low.max_wait_time_estimate), + }, + medium: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.medium.suggested_max_fee_per_gas)?, + max_priority_fee_per_gas: wei_from_gwei_decimal!( + &infura_fees.medium.suggested_max_priority_fee_per_gas + )?, + min_wait_time: Some(infura_fees.medium.min_wait_time_estimate), + max_wait_time: Some(infura_fees.medium.max_wait_time_estimate), + }, + high: FeePerGasLevel { + max_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.high.suggested_max_fee_per_gas)?, + max_priority_fee_per_gas: wei_from_gwei_decimal!(&infura_fees.high.suggested_max_priority_fee_per_gas)?, + min_wait_time: Some(infura_fees.high.min_wait_time_estimate), + max_wait_time: Some(infura_fees.high.max_wait_time_estimate), + }, + source: EstimationSource::Infura, + base_fee_trend: infura_fees.base_fee_trend, + priority_fee_trend: infura_fees.priority_fee_trend, + }) + } +} + +/// Infura gas api provider caller +#[allow(dead_code)] +pub(crate) struct InfuraGasApiCaller {} + +#[allow(dead_code)] +impl InfuraGasApiCaller { + const INFURA_GAS_FEES_ENDPOINT: &'static str = "networks/1/suggestedGasFees"; // Support only main chain + + fn get_infura_gas_api_url(base_url: &Url) -> (Url, Vec<(&'static str, &'static str)>) { + let mut url = base_url.clone(); + url.set_path(Self::INFURA_GAS_FEES_ENDPOINT); + let headers = vec![("Authorization", INFURA_GAS_API_AUTH_TEST.as_str())]; + (url, headers) + } + + async fn make_infura_gas_api_request( + url: &Url, + headers: Vec<(&'static str, &'static str)>, + ) -> Result> { + let resp = slurp_url_with_headers(url.as_str(), headers) + .await + .mm_err(|e| e.to_string())?; + if resp.0 != StatusCode::OK { + let error = format!("{} failed with status code {}", url, resp.0); + return MmError::err(error); + } + let estimated_fees = json::from_slice(&resp.2).map_to_mm(|e| e.to_string())?; + Ok(estimated_fees) + } + + /// Fetch fee per gas estimations from infura provider + pub async fn fetch_infura_fee_estimation(base_url: &Url) -> Web3RpcResult { + let (url, headers) = Self::get_infura_gas_api_url(base_url); + let infura_estimated_fees = Self::make_infura_gas_api_request(&url, headers) + .await + .mm_err(Web3RpcError::Transport)?; + infura_estimated_fees.try_into().mm_err(Into::into) + } +} diff --git a/mm2src/coins/eth/fee_estimation/eip1559/mod.rs b/mm2src/coins/eth/fee_estimation/eip1559/mod.rs new file mode 100644 index 0000000000..3dac4c54bd --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/eip1559/mod.rs @@ -0,0 +1,116 @@ +//! Provides estimations of base and priority fee per gas or fetch estimations from a gas api provider +pub mod block_native; +pub mod infura; +pub mod simple; + +use crate::AsyncMutex; +use common::executor::AbortOnDropHandle; + +use ethereum_types::U256; +use std::sync::Arc; +use url::Url; + +const FEE_PER_GAS_LEVELS: usize = 3; + +/// Indicates which provider was used to get fee per gas estimations +#[derive(Clone, Debug)] +pub enum EstimationSource { + /// filled by default values + Empty, + /// internal simple estimator + Simple, + Infura, + Blocknative, +} + +impl ToString for EstimationSource { + fn to_string(&self) -> String { + match self { + EstimationSource::Empty => "empty".into(), + EstimationSource::Simple => "simple".into(), + EstimationSource::Infura => "infura".into(), + EstimationSource::Blocknative => "blocknative".into(), + } + } +} + +impl Default for EstimationSource { + fn default() -> Self { Self::Empty } +} + +enum PriorityLevelId { + Low = 0, + Medium = 1, + High = 2, +} + +/// Supported gas api providers +#[derive(Clone, Deserialize)] +pub enum GasApiProvider { + Infura, + Blocknative, +} + +#[derive(Clone, Deserialize)] +pub struct GasApiConfig { + /// gas api provider name to use + pub provider: GasApiProvider, + /// gas api provider or proxy base url (scheme, host and port without the relative part) + pub url: Url, +} + +/// Priority level estimated max fee per gas +#[derive(Clone, Debug, Default)] +pub struct FeePerGasLevel { + /// estimated max priority tip fee per gas in wei + pub max_priority_fee_per_gas: U256, + /// estimated max fee per gas in wei + pub max_fee_per_gas: U256, + /// estimated transaction min wait time in mempool in ms for this priority level + pub min_wait_time: Option, + /// estimated transaction max wait time in mempool in ms for this priority level + pub max_wait_time: Option, +} + +/// Internal struct for estimated fee per gas for several priority levels, in wei +/// low/medium/high levels are supported +#[derive(Default, Debug, Clone)] +pub struct FeePerGasEstimated { + /// base fee for the next block in wei + pub base_fee: U256, + /// estimated low priority fee + pub low: FeePerGasLevel, + /// estimated medium priority fee + pub medium: FeePerGasLevel, + /// estimated high priority fee + pub high: FeePerGasLevel, + /// which estimator used + pub source: EstimationSource, + /// base trend (up or down) + pub base_fee_trend: String, + /// priority trend (up or down) + pub priority_fee_trend: String, +} + +/// Gas fee estimator loop context, runs a loop to estimate max fee and max priority fee per gas according to EIP-1559 for the next block +/// +/// This FeeEstimatorContext handles rpc requests which start and stop gas fee estimation loop and handles the loop itself. +/// FeeEstimatorContext keeps the latest estimated gas fees to return them on rpc request +pub(crate) struct FeeEstimatorContext { + /// Latest estimated gas fee values + pub(crate) estimated_fees: Arc>, + /// Handler for estimator loop graceful shutdown + pub(crate) abort_handler: AsyncMutex>, +} + +/// Gas fee estimator creation state +pub(crate) enum FeeEstimatorState { + /// Gas fee estimation not supported for this coin + CoinNotSupported, + /// Platform coin required to be enabled for gas fee estimation for this coin + PlatformCoinRequired, + /// Fee estimator created, use simple internal estimator + Simple(AsyncMutex), + /// Fee estimator created, use provider or simple internal estimator (if provider fails) + Provider(AsyncMutex), +} diff --git a/mm2src/coins/eth/fee_estimation/eip1559/simple.rs b/mm2src/coins/eth/fee_estimation/eip1559/simple.rs new file mode 100644 index 0000000000..c6e1c3513d --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/eip1559/simple.rs @@ -0,0 +1,137 @@ +use super::{EstimationSource, FeePerGasEstimated, FeePerGasLevel, PriorityLevelId, FEE_PER_GAS_LEVELS}; +use crate::eth::web3_transport::FeeHistoryResult; +use crate::eth::{Web3RpcError, Web3RpcResult}; +use crate::{wei_from_gwei_decimal, wei_to_gwei_decimal, EthCoin}; +use mm2_err_handle::mm_error::MmError; +use mm2_err_handle::or_mm_error::OrMmError; +use mm2_number::BigDecimal; + +use ethereum_types::U256; +use num_traits::FromPrimitive; +use web3::types::BlockNumber; + +/// Simple priority fee per gas estimator based on fee history +/// normally used if gas api provider is not available +pub(crate) struct FeePerGasSimpleEstimator {} + +impl FeePerGasSimpleEstimator { + // TODO: add minimal max fee and priority fee + /// depth to look for fee history to estimate priority fees + const FEE_PRIORITY_DEPTH: u64 = 5u64; + + /// percentiles to pass to eth_feeHistory + const HISTORY_PERCENTILES: [f64; FEE_PER_GAS_LEVELS] = [25.0, 50.0, 75.0]; + + /// percentile to predict next base fee over historical rewards + const BASE_FEE_PERCENTILE: f64 = 75.0; + + /// percentiles to calc max priority fee over historical rewards + const PRIORITY_FEE_PERCENTILES: [f64; FEE_PER_GAS_LEVELS] = [50.0, 50.0, 50.0]; + + /// adjustment for max fee per gas picked up by sampling + const ADJUST_MAX_FEE: [f64; FEE_PER_GAS_LEVELS] = [1.1, 1.175, 1.25]; // 1.25 assures max_fee_per_gas will be over next block base_fee + + /// adjustment for max priority fee picked up by sampling + const ADJUST_MAX_PRIORITY_FEE: [f64; FEE_PER_GAS_LEVELS] = [1.0, 1.0, 1.0]; + + /// block depth for eth_feeHistory + pub fn history_depth() -> u64 { Self::FEE_PRIORITY_DEPTH } + + /// percentiles for priority rewards obtained with eth_feeHistory + pub fn history_percentiles() -> &'static [f64] { &Self::HISTORY_PERCENTILES } + + /// percentile for vector + fn percentile_of(v: &[U256], percent: f64) -> U256 { + let mut v_mut = v.to_owned(); + v_mut.sort(); + + // validate bounds: + let percent = if percent > 100.0 { 100.0 } else { percent }; + let percent = if percent < 0.0 { 0.0 } else { percent }; + + let value_pos = ((v_mut.len() - 1) as f64 * percent / 100.0).round() as usize; + v_mut[value_pos] + } + + /// Estimate simplified gas priority fees based on fee history + pub async fn estimate_fee_by_history(coin: &EthCoin) -> Web3RpcResult { + let res: Result = coin + .eth_fee_history( + U256::from(Self::history_depth()), + BlockNumber::Latest, + Self::history_percentiles(), + ) + .await; + + match res { + Ok(fee_history) => Ok(Self::calculate_with_history(&fee_history)?), + Err(_) => MmError::err(Web3RpcError::Internal("Eth requests failed".into())), + } + } + + fn predict_base_fee(base_fees: &[U256]) -> U256 { Self::percentile_of(base_fees, Self::BASE_FEE_PERCENTILE) } + + fn priority_fee_for_level( + level: PriorityLevelId, + base_fee: BigDecimal, + fee_history: &FeeHistoryResult, + ) -> Web3RpcResult { + let level_index = level as usize; + let level_rewards = fee_history + .priority_rewards + .as_ref() + .or_mm_err(|| Web3RpcError::Internal("expected reward in eth_feeHistory".into()))? + .iter() + .map(|rewards| rewards.get(level_index).copied().unwrap_or_else(|| U256::from(0))) + .collect::>(); + + // Calculate the max priority fee per gas based on the rewards percentile. + let max_priority_fee_per_gas = Self::percentile_of(&level_rewards, Self::PRIORITY_FEE_PERCENTILES[level_index]); + // Convert the priority fee to BigDecimal gwei, falling back to 0 on error. + let max_priority_fee_per_gas_gwei = + wei_to_gwei_decimal!(max_priority_fee_per_gas).unwrap_or_else(|_| BigDecimal::from(0)); + + // Calculate the max fee per gas by adjusting the base fee and adding the priority fee. + let adjust_max_fee = + BigDecimal::from_f64(Self::ADJUST_MAX_FEE[level_index]).unwrap_or_else(|| BigDecimal::from(0)); + let adjust_max_priority_fee = + BigDecimal::from_f64(Self::ADJUST_MAX_PRIORITY_FEE[level_index]).unwrap_or_else(|| BigDecimal::from(0)); + + // TODO: consider use checked ops + let max_fee_per_gas_dec = base_fee * adjust_max_fee + max_priority_fee_per_gas_gwei * adjust_max_priority_fee; + + Ok(FeePerGasLevel { + max_priority_fee_per_gas, + max_fee_per_gas: wei_from_gwei_decimal!(&max_fee_per_gas_dec)?, + // TODO: Consider adding default wait times if applicable (and mark them as uncertain). + min_wait_time: None, + max_wait_time: None, + }) + } + + /// estimate priority fees by fee history + fn calculate_with_history(fee_history: &FeeHistoryResult) -> Web3RpcResult { + // For estimation of max fee and max priority fee we use latest block base_fee but adjusted. + // Apparently for this simple fee estimator for assured high priority we should assume + // that the real base_fee may go up by 1,25 (i.e. if the block is full). This is covered by high priority ADJUST_MAX_FEE multiplier + let latest_base_fee = fee_history + .base_fee_per_gas + .first() + .cloned() + .unwrap_or_else(|| U256::from(0)); + let latest_base_fee_dec = wei_to_gwei_decimal!(latest_base_fee).unwrap_or_else(|_| BigDecimal::from(0)); + + // The predicted base fee is not used for calculating eip1559 values here and is provided for other purposes + // (f.e if the caller would like to do own estimates of max fee and max priority fee) + let predicted_base_fee = Self::predict_base_fee(&fee_history.base_fee_per_gas); + Ok(FeePerGasEstimated { + base_fee: predicted_base_fee, + low: Self::priority_fee_for_level(PriorityLevelId::Low, latest_base_fee_dec.clone(), fee_history)?, + medium: Self::priority_fee_for_level(PriorityLevelId::Medium, latest_base_fee_dec.clone(), fee_history)?, + high: Self::priority_fee_for_level(PriorityLevelId::High, latest_base_fee_dec, fee_history)?, + source: EstimationSource::Simple, + base_fee_trend: String::default(), + priority_fee_trend: String::default(), + }) + } +} diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs new file mode 100644 index 0000000000..badabf80dd --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -0,0 +1,175 @@ +use super::eip1559::GasApiConfig; +use crate::eth::EthCoin; +use common::executor::Timer; +use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; + +use async_trait::async_trait; +use futures::channel::oneshot; +use instant::Instant; +use serde::Deserialize; +use serde_json::Value as Json; +use std::convert::TryFrom; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields, default)] +struct EthFeeStreamingConfig { + /// The time in seconds to wait before re-estimating the gas fees. + pub estimate_every: f64, + /// Optional Gas API provider to be used to fetch gas prices. + /// If not provided, the "simple" (history based) provider will be used. + // FIXME: There is a similar field in MM2.json config, that one is the one used for swaps. + // Should we allow setting the one in the streamer like this? More flexible but will end + // up confusing users. What about setting `gas_api` with another end point and always use + // the one set. + pub gas_api_provider: Option, +} + +impl Default for EthFeeStreamingConfig { + fn default() -> Self { + Self { + estimate_every: 15.0, + gas_api_provider: None, + } + } +} + +pub struct EthFeeEventStreamer { + config: EthFeeStreamingConfig, + coin: EthCoin, +} + +impl EthFeeEventStreamer { + pub fn try_new(config: Option, coin: EthCoin) -> serde_json::Result { + let config = config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?; + + Ok(Self { config, coin }) + } +} + +#[async_trait] +impl EventStreamer for EthFeeEventStreamer { + type DataInType = NoDataIn; + + fn streamer_id(&self) -> String { format!("FEE_ESTIMATION:{}", self.coin.ticker) } + + async fn handle( + self, + broadcaster: StreamingManager, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + loop { + let now = Instant::now(); + match self + .coin + .get_eip1559_gas_fee(self.config.gas_api_provider.clone()) + .await + .map(serialized::FeePerGasEstimated::try_from) + { + Ok(Ok(fee)) => { + let fee = serde_json::to_value(fee).expect("Serialization shouldn't fail"); + broadcaster.broadcast(Event::new(self.streamer_id(), fee)); + }, + Ok(Err(err)) => { + let err = json!({ "error": err.to_string() }); + broadcaster.broadcast(Event::err(self.streamer_id(), err)); + }, + Err(err) => { + let err = serde_json::to_value(err).expect("Serialization shouldn't fail"); + broadcaster.broadcast(Event::err(self.streamer_id(), err)); + }, + } + let sleep_time = self.config.estimate_every - now.elapsed().as_secs_f64(); + if sleep_time >= 0.1 { + Timer::sleep(sleep_time).await; + } + } + } +} + +/// Serializable version of fee estimation data. +mod serialized { + use crate::eth::fee_estimation::eip1559; + use crate::{wei_to_gwei_decimal, NumConversError}; + use mm2_err_handle::mm_error::MmError; + use mm2_number::BigDecimal; + + use std::convert::TryFrom; + + /// Estimated fee per gas units + #[derive(Serialize)] + pub enum EstimationUnits { + Gwei, + } + + /// Priority level estimated max fee per gas + #[derive(Serialize)] + pub struct FeePerGasLevel { + /// estimated max priority tip fee per gas in gwei + pub max_priority_fee_per_gas: BigDecimal, + /// estimated max fee per gas in gwei + pub max_fee_per_gas: BigDecimal, + /// estimated transaction min wait time in mempool in ms for this priority level + pub min_wait_time: Option, + /// estimated transaction max wait time in mempool in ms for this priority level + pub max_wait_time: Option, + } + + /// External struct for estimated fee per gas for several priority levels, in gwei + /// low/medium/high levels are supported + #[derive(Serialize)] + pub struct FeePerGasEstimated { + /// base fee for the next block in gwei + pub base_fee: BigDecimal, + /// estimated low priority fee + pub low: FeePerGasLevel, + /// estimated medium priority fee + pub medium: FeePerGasLevel, + /// estimated high priority fee + pub high: FeePerGasLevel, + /// which estimator used + pub source: String, + /// base trend (up or down) + pub base_fee_trend: String, + /// priority trend (up or down) + pub priority_fee_trend: String, + /// fee units + pub units: EstimationUnits, + } + + impl TryFrom for FeePerGasEstimated { + type Error = MmError; + + fn try_from(fees: eip1559::FeePerGasEstimated) -> Result { + Ok(Self { + base_fee: wei_to_gwei_decimal!(fees.base_fee)?, + low: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_priority_fee_per_gas)?, + min_wait_time: fees.low.min_wait_time, + max_wait_time: fees.low.max_wait_time, + }, + medium: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_priority_fee_per_gas)?, + min_wait_time: fees.medium.min_wait_time, + max_wait_time: fees.medium.max_wait_time, + }, + high: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_priority_fee_per_gas)?, + min_wait_time: fees.high.min_wait_time, + max_wait_time: fees.high.max_wait_time, + }, + source: fees.source.to_string(), + base_fee_trend: fees.base_fee_trend, + priority_fee_trend: fees.priority_fee_trend, + units: EstimationUnits::Gwei, + }) + } + } +} diff --git a/mm2src/coins/eth/fee_estimation/mod.rs b/mm2src/coins/eth/fee_estimation/mod.rs new file mode 100644 index 0000000000..f519e95369 --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod eip1559; +pub mod eth_fee_events; diff --git a/mm2src/coins/rpc_command/get_estimated_fees.rs b/mm2src/coins/rpc_command/get_estimated_fees.rs index b62e572756..cbe85b2df4 100644 --- a/mm2src/coins/rpc_command/get_estimated_fees.rs +++ b/mm2src/coins/rpc_command/get_estimated_fees.rs @@ -1,6 +1,6 @@ //! RPCs to start/stop gas fee estimator and get estimated base and priority fee per gas - -use crate::eth::{EthCoin, EthCoinType, FeeEstimatorContext, FeeEstimatorState, FeePerGasEstimated}; +use crate::eth::fee_estimation::eip1559::{FeeEstimatorContext, FeeEstimatorState, FeePerGasEstimated}; +use crate::eth::{EthCoin, EthCoinType}; use crate::{lp_coinfind_or_err, wei_to_gwei_decimal, AsyncMutex, CoinFindError, MmCoinEnum, NumConversError}; use common::executor::{spawn_abortable, Timer}; use common::log::debug; @@ -8,6 +8,7 @@ use common::{HttpStatusCode, StatusCode}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use mm2_number::BigDecimal; + use serde::{Deserialize, Serialize}; use serde_json::{self as json, Value as Json}; use std::convert::{TryFrom, TryInto}; @@ -262,7 +263,7 @@ impl FeeEstimatorContext { loop { let started = common::now_float(); if let Ok(estimator_ctx) = Self::get_estimator_ctx(&coin) { - let estimated_fees = coin.get_eip1559_gas_fee().await.unwrap_or_default(); + let estimated_fees = coin.get_eip1559_gas_fee(None).await.unwrap_or_default(); let estimator_ctx = estimator_ctx.lock().await; *estimator_ctx.estimated_fees.lock().await = estimated_fees; } diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index efb99fd3ca..3aaa6fff23 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -28,7 +28,7 @@ impl HeartbeatEvent { pub fn try_new(config: Option) -> serde_json::Result { Ok(Self { config: config - .map(|c| serde_json::from_value(c)) + .map(serde_json::from_value) .unwrap_or(Ok(Default::default()))?, }) } diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index a7c4551625..d83a67f767 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -219,9 +219,10 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, ibc_chains).await, "ibc_transfer_channels" => handle_mmrpc(ctx, request, ibc_transfer_channels).await, "withdraw_nft" => handle_mmrpc(ctx, request, withdraw_nft).await, - "start_eth_fee_estimator" => handle_mmrpc(ctx, request, start_eth_fee_estimator).await, - "stop_eth_fee_estimator" => handle_mmrpc(ctx, request, stop_eth_fee_estimator).await, - "get_eth_estimated_fee_per_gas" => handle_mmrpc(ctx, request, get_eth_estimated_fee_per_gas).await, + // FIXME: Remove these methods? + // "start_eth_fee_estimator" => handle_mmrpc(ctx, request, start_eth_fee_estimator).await, + // "stop_eth_fee_estimator" => handle_mmrpc(ctx, request, stop_eth_fee_estimator).await, + // "get_eth_estimated_fee_per_gas" => handle_mmrpc(ctx, request, get_eth_estimated_fee_per_gas).await, "get_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, get_swap_transaction_fee_policy).await, "set_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, set_swap_transaction_fee_policy).await, "send_asked_data" => handle_mmrpc(ctx, request, send_asked_data_rpc).await, @@ -336,8 +337,8 @@ async fn rpc_streaming_dispatcher( "network::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_network_streaming).await, "heartbeat::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_heartbeat_streaming).await, "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat_streaming).await, - // "eth_fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_eth_fee_estimation_streaming).await, - // "eth_fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_eth_fee_estimation_streaming).await, + "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation_streaming).await, + "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation_streaming).await, // "swap_status::enable" => handle_mmrpc(ctx, request, enable_swap_status_streaming).await, // "swap_status::disable" => handle_mmrpc(ctx, request, disable_swap_status_streaming).await, // "order_status::enable" => handle_mmrpc(ctx, request, enable_order_status_streaming).await, diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs new file mode 100644 index 0000000000..f037b31912 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -0,0 +1,72 @@ +//! RPC activation and deactivation for different fee estimation streamers. +use super::{DisableStreamingRequest, EnableStreamingResponse}; + +use coins::eth::fee_estimation::eth_fee_events::EthFeeEventStreamer; +use coins::{lp_coinfind, MmCoinEnum}; +use common::HttpStatusCode; +use http::StatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use serde_json::Value as Json; + +#[derive(Deserialize)] +pub struct EnableFeeStreamingRequest { + pub client_id: u64, + pub coin: String, + pub config: Option, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum FeeStreamingRequestError { + EnableError(String), + DisableError(String), + CoinNotFound, + CoinNotSupported, + Internal(String), +} + +impl HttpStatusCode for FeeStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + FeeStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + FeeStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + FeeStreamingRequestError::CoinNotFound => StatusCode::NOT_FOUND, + FeeStreamingRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, + FeeStreamingRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn enable_fee_estimation_streaming( + ctx: MmArc, + req: EnableFeeStreamingRequest, +) -> MmResult { + let coin = lp_coinfind(&ctx, &req.coin) + .await + .map_err(FeeStreamingRequestError::Internal)? + .ok_or(FeeStreamingRequestError::CoinNotFound)?; + + match coin { + MmCoinEnum::EthCoin(coin) => { + let eth_fee_estimator_streamer = EthFeeEventStreamer::try_new(req.config, coin) + .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}")))?; + ctx.event_stream_manager + .add(req.client_id, eth_fee_estimator_streamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}"))) + }, + _ => Err(FeeStreamingRequestError::CoinNotSupported)?, + } +} + +pub async fn disable_fee_estimation_streaming( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult<(), FeeStreamingRequestError> { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| FeeStreamingRequestError::DisableError(format!("{e:?}"))) +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index ee94c17ef8..fa749f2119 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -1,31 +1,29 @@ mod balance; -mod network; +mod fee_estimation; mod heartbeat; -mod eth_fee_estimator; +mod network; // Re-exports pub use balance::*; -pub use network::*; +pub use fee_estimation::*; pub use heartbeat::*; -pub use eth_fee_estimator::*; +pub use network::*; #[derive(Serialize)] -/// The success response for any event streaming activation endpoint. +/// The success/ok response for any event streaming activation request. pub struct EnableStreamingResponse { pub streamer_id: String, + // FIXME: Consider returning the applied config here (might be different from the one the client requested). + // pub config: Json, } impl EnableStreamingResponse { - fn new(streamer_id: String) -> Self { - Self { - streamer_id - } - } + fn new(streamer_id: String) -> Self { Self { streamer_id } } } #[derive(Deserialize)] -/// The request used for any event streaming deactivation endpoint. +/// The request used for any event streaming deactivation. pub struct DisableStreamingRequest { pub client_id: u64, pub streamer_id: String, -} \ No newline at end of file +} diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index c877c1ca28..1f45552507 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -1,10 +1,11 @@ use crate::p2p::P2PContext; -use async_trait::async_trait; use common::executor::Timer; -use futures::channel::oneshot; use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; use mm2_libp2p::behaviours::atomicdex; + +use async_trait::async_trait; +use futures::channel::oneshot; use serde::Deserialize; use serde_json::{json, Value as Json}; @@ -34,9 +35,7 @@ pub struct NetworkEvent { impl NetworkEvent { pub fn try_new(config: Option, ctx: MmArc) -> serde_json::Result { Ok(Self { - config: config - .map(|c| serde_json::from_value(c)) - .unwrap_or(Ok(Default::default()))?, + config: config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?, ctx, }) } From 8d8d496cdb0aa3e3c961ec53da13272d5f339414 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 25 Jul 2024 11:55:40 +0200 Subject: [PATCH 057/128] remove _streaming postfix from streaming fn names they are already in a streaming_activations module (already tells what these functions are for), so cut the names short --- .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 24 +++++++++---------- .../src/rpc/streaming_activations/balance.rs | 4 ++-- .../streaming_activations/fee_estimation.rs | 4 ++-- .../rpc/streaming_activations/heartbeat.rs | 4 ++-- .../src/rpc/streaming_activations/network.rs | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index d83a67f767..af9f328fa4 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -331,18 +331,18 @@ async fn rpc_streaming_dispatcher( streaming_request: String, ) -> DispatcherResult>> { match streaming_request.as_str() { - "balance::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_balance_streaming).await, - "balance::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_balance_streaming).await, - "network::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_network_streaming).await, - "network::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_network_streaming).await, - "heartbeat::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_heartbeat_streaming).await, - "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat_streaming).await, - "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation_streaming).await, - "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation_streaming).await, - // "swap_status::enable" => handle_mmrpc(ctx, request, enable_swap_status_streaming).await, - // "swap_status::disable" => handle_mmrpc(ctx, request, disable_swap_status_streaming).await, - // "order_status::enable" => handle_mmrpc(ctx, request, enable_order_status_streaming).await, - // "order_status::disable" => handle_mmrpc(ctx, request, disable_order_status_streaming).await, + "balance::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_balance).await, + "balance::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_balance).await, + "network::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_network).await, + "network::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_network).await, + "heartbeat::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_heartbeat).await, + "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat).await, + "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation).await, + "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation).await, + // "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, + // "swap_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_swap_status).await, + // "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, + // "order_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_order_status).await, _ => MmError::err(DispatcherError::NoSuchMethod), } } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 959a7a6971..04d6c5bff5 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -42,7 +42,7 @@ impl HttpStatusCode for BalanceStreamingRequestError { } } -pub async fn enable_balance_streaming( +pub async fn enable_balance( ctx: MmArc, req: EnableBalanceStreamingRequest, ) -> MmResult { @@ -123,7 +123,7 @@ pub async fn enable_balance_streaming( .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}"))) } -pub async fn disable_balance_streaming( +pub async fn disable_balance( ctx: MmArc, req: DisableStreamingRequest, ) -> MmResult<(), BalanceStreamingRequestError> { diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index f037b31912..c23b0ef44f 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -39,7 +39,7 @@ impl HttpStatusCode for FeeStreamingRequestError { } } -pub async fn enable_fee_estimation_streaming( +pub async fn enable_fee_estimation( ctx: MmArc, req: EnableFeeStreamingRequest, ) -> MmResult { @@ -62,7 +62,7 @@ pub async fn enable_fee_estimation_streaming( } } -pub async fn disable_fee_estimation_streaming( +pub async fn disable_fee_estimation( ctx: MmArc, req: DisableStreamingRequest, ) -> MmResult<(), FeeStreamingRequestError> { diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index bbc63ceff9..0a43079e62 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -31,7 +31,7 @@ impl HttpStatusCode for HeartbeatRequestError { } } -pub async fn enable_heartbeat_streaming( +pub async fn enable_heartbeat( ctx: MmArc, req: EnableHeartbeatRequest, ) -> MmResult { @@ -44,7 +44,7 @@ pub async fn enable_heartbeat_streaming( .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}"))) } -pub async fn disable_heartbeat_streaming( +pub async fn disable_heartbeat( ctx: MmArc, req: DisableStreamingRequest, ) -> MmResult<(), HeartbeatRequestError> { diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs index 546e1a4275..96fb734b93 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -31,7 +31,7 @@ impl HttpStatusCode for NetworkStreamingRequestError { } } -pub async fn enable_network_streaming( +pub async fn enable_network( ctx: MmArc, req: EnableNetworkStreamingRequest, ) -> MmResult { @@ -44,7 +44,7 @@ pub async fn enable_network_streaming( .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}"))) } -pub async fn disable_network_streaming( +pub async fn disable_network( ctx: MmArc, req: DisableStreamingRequest, ) -> MmResult<(), NetworkStreamingRequestError> { From 0859434158970feb1ece8d074268bf2eb4273397 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 25 Jul 2024 12:26:27 +0200 Subject: [PATCH 058/128] send a non-empty message (success) on successfully rpc call also fixes a bug where the eth fee streamer was spawned on the ctx weakspawner --- .../src/rpc/streaming_activations/balance.rs | 7 ++++--- .../rpc/streaming_activations/fee_estimation.rs | 13 +++++++------ .../src/rpc/streaming_activations/heartbeat.rs | 7 ++++--- .../mm2_main/src/rpc/streaming_activations/mod.rs | 15 +++++++++++++++ .../src/rpc/streaming_activations/network.rs | 7 ++++--- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 04d6c5bff5..8ec8b9aaad 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different balance event streamers. -use super::{DisableStreamingRequest, EnableStreamingResponse}; +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; use coins::eth::eth_balance_events::EthBalanceEventStreamer; use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; @@ -126,8 +126,9 @@ pub async fn enable_balance( pub async fn disable_balance( ctx: MmArc, req: DisableStreamingRequest, -) -> MmResult<(), BalanceStreamingRequestError> { +) -> MmResult { ctx.event_stream_manager .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| BalanceStreamingRequestError::DisableError(format!("{e:?}"))) + .map_to_mm(|e| BalanceStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index c23b0ef44f..9a46a88c06 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -1,8 +1,8 @@ //! RPC activation and deactivation for different fee estimation streamers. -use super::{DisableStreamingRequest, EnableStreamingResponse}; +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; use coins::eth::fee_estimation::eth_fee_events::EthFeeEventStreamer; -use coins::{lp_coinfind, MmCoinEnum}; +use coins::{lp_coinfind, MmCoin, MmCoinEnum}; use common::HttpStatusCode; use http::StatusCode; use mm2_core::mm_ctx::MmArc; @@ -50,10 +50,10 @@ pub async fn enable_fee_estimation( match coin { MmCoinEnum::EthCoin(coin) => { - let eth_fee_estimator_streamer = EthFeeEventStreamer::try_new(req.config, coin) + let eth_fee_estimator_streamer = EthFeeEventStreamer::try_new(req.config, coin.clone()) .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}")))?; ctx.event_stream_manager - .add(req.client_id, eth_fee_estimator_streamer, ctx.spawner()) + .add(req.client_id, eth_fee_estimator_streamer, coin.spawner()) .await .map(EnableStreamingResponse::new) .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}"))) @@ -65,8 +65,9 @@ pub async fn enable_fee_estimation( pub async fn disable_fee_estimation( ctx: MmArc, req: DisableStreamingRequest, -) -> MmResult<(), FeeStreamingRequestError> { +) -> MmResult { ctx.event_stream_manager .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| FeeStreamingRequestError::DisableError(format!("{e:?}"))) + .map_to_mm(|e| FeeStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index 0a43079e62..3431397fb6 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the heartbeats. -use super::{DisableStreamingRequest, EnableStreamingResponse}; +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; use crate::mm2::heartbeat_event::HeartbeatEvent; use common::HttpStatusCode; @@ -47,8 +47,9 @@ pub async fn enable_heartbeat( pub async fn disable_heartbeat( ctx: MmArc, req: DisableStreamingRequest, -) -> MmResult<(), HeartbeatRequestError> { +) -> MmResult { ctx.event_stream_manager .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| HeartbeatRequestError::DisableError(format!("{e:?}"))) + .map_to_mm(|e| HeartbeatRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index fa749f2119..7d96884081 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -2,15 +2,20 @@ mod balance; mod fee_estimation; mod heartbeat; mod network; +// mod orders; +// mod swaps; // Re-exports pub use balance::*; pub use fee_estimation::*; pub use heartbeat::*; pub use network::*; +// pub use orders::*; +// pub use swaps::*; #[derive(Serialize)] /// The success/ok response for any event streaming activation request. +/// Note that we don't have a unified EnableStreamingRequest. It is defined per event streaming activation. pub struct EnableStreamingResponse { pub streamer_id: String, // FIXME: Consider returning the applied config here (might be different from the one the client requested). @@ -27,3 +32,13 @@ pub struct DisableStreamingRequest { pub client_id: u64, pub streamer_id: String, } + +#[derive(Serialize)] +/// The success/ok response for any event streaming deactivation request. +pub struct DisableStreamingResponse { + result: &'static str, +} + +impl DisableStreamingResponse { + fn new() -> Self { Self { result: "Success" } } +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs index 96fb734b93..8a95f5b76b 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the network event streamer. -use super::{DisableStreamingRequest, EnableStreamingResponse}; +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; use common::HttpStatusCode; use http::StatusCode; @@ -47,8 +47,9 @@ pub async fn enable_network( pub async fn disable_network( ctx: MmArc, req: DisableStreamingRequest, -) -> MmResult<(), NetworkStreamingRequestError> { +) -> MmResult { ctx.event_stream_manager .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| NetworkStreamingRequestError::DisableError(format!("{e:?}"))) + .map_to_mm(|e| NetworkStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) } From eec1f153e44e1ca343d8868b32f78445bca7637a Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 25 Jul 2024 20:30:39 +0200 Subject: [PATCH 059/128] stream swap status in real time this only streams the specific event that took place along with the UUID since most events don't embed the UUID. This doesn't work like `swap_v2_rpcs::SwapRpcData` where all the past events are send in a vector --- mm2src/coins/utxo/utxo_balance_events.rs | 2 +- .../src/tendermint_with_assets_activation.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 2 + mm2src/mm2_main/src/heartbeat_event.rs | 4 +- mm2src/mm2_main/src/lp_swap.rs | 1 + mm2src/mm2_main/src/lp_swap/maker_swap.rs | 12 ++++- mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs | 19 ++++--- mm2src/mm2_main/src/lp_swap/swap_events.rs | 51 +++++++++++++++++++ mm2src/mm2_main/src/lp_swap/swap_v2_common.rs | 2 +- mm2src/mm2_main/src/lp_swap/taker_swap.rs | 13 ++++- mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs | 19 ++++--- mm2src/mm2_main/src/rpc.rs | 3 +- .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 5 +- .../rpc/streaming_activations/heartbeat.rs | 4 +- .../src/rpc/streaming_activations/mod.rs | 4 +- .../src/rpc/streaming_activations/swaps.rs | 51 +++++++++++++++++++ 16 files changed, 165 insertions(+), 29 deletions(-) create mode 100644 mm2src/mm2_main/src/lp_swap/swap_events.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 916f7bc0ab..1036969369 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use common::{executor::Timer, log, Future01CompatExt}; use futures::channel::oneshot; -use futures_util::StreamExt; +use futures::StreamExt; use keys::Address; use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; use std::collections::{BTreeMap, HashSet}; diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 4f12154dbf..34bf3bdfed 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -25,7 +25,7 @@ use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::{Value as Json}; +use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; impl TokenOf for TendermintToken { diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 277c44ef64..8c50efec16 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -142,6 +142,8 @@ impl StreamingManager { } /// Sends data to a streamer with `streamer_id`. + // FIXME: Consider receiving `data` as a `Clone`able &T and only clone it if streamer is found. + // This is to reduce the memory overhead when streamers are inactive. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { let streamers = self.streamers.read().unwrap(); let streamer_info = streamers diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 3aaa6fff23..bccd9a876e 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -27,9 +27,7 @@ pub struct HeartbeatEvent { impl HeartbeatEvent { pub fn try_new(config: Option) -> serde_json::Result { Ok(Self { - config: config - .map(serde_json::from_value) - .unwrap_or(Ok(Default::default()))?, + config: config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?, }) } } diff --git a/mm2src/mm2_main/src/lp_swap.rs b/mm2src/mm2_main/src/lp_swap.rs index 272b53c0dc..57ec67e300 100644 --- a/mm2src/mm2_main/src/lp_swap.rs +++ b/mm2src/mm2_main/src/lp_swap.rs @@ -106,6 +106,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; #[path = "lp_swap/komodefi.swap_v2.pb.rs"] #[rustfmt::skip] mod swap_v2_pb; +#[path = "lp_swap/swap_events.rs"] pub(crate) mod swap_events; #[path = "lp_swap/swap_v2_common.rs"] mod swap_v2_common; #[path = "lp_swap/swap_v2_rpcs.rs"] pub(crate) mod swap_v2_rpcs; #[path = "lp_swap/swap_watcher.rs"] pub(crate) mod swap_watcher; diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index 2594d614ad..af76e7e8e5 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -13,6 +13,7 @@ use super::{broadcast_my_swap_status, broadcast_p2p_tx_msg, broadcast_swap_msg_e use crate::mm2::lp_dispatcher::{DispatcherContext, LpEvents}; use crate::mm2::lp_network::subscribe_to_topic; use crate::mm2::lp_ordermatch::MakerOrderBuilder; +use crate::mm2::lp_swap::swap_events::{SwapStatusEvent, SwapStatusStreamer}; use crate::mm2::lp_swap::swap_v2_common::mark_swap_as_finished; use crate::mm2::lp_swap::{broadcast_swap_message, taker_payment_spend_duration, MAX_STARTED_AT_DIFF}; use coins::lp_price::fetch_swap_coins_price; @@ -29,6 +30,7 @@ use futures::{compat::Future01CompatExt, select, FutureExt}; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; use mm2_rpc::data::legacy::OrderConfirmationsSettings; use parking_lot::Mutex as PaMutex; @@ -2113,10 +2115,16 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { .dispatch_async(ctx.clone(), LpEvents::MakerSwapStatusChanged(event_to_send)) .await; drop(dispatcher); - save_my_maker_swap_event(&ctx, &running_swap, to_save) + save_my_maker_swap_event(&ctx, &running_swap, to_save.clone()) .await .expect("!save_my_maker_swap_event"); - // FIXME: send SSE here + // Send a notification to the swap status streamer about a new event. + ctx.event_stream_manager + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::MakerV1 { + uuid: running_swap.uuid, + event: to_save, + }) + .ok(); if event.should_ban_taker() { ban_pubkey_on_failed_swap( &ctx, diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs index 6e9555c10f..0fc5a4ca72 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs @@ -1,3 +1,4 @@ +use super::swap_events::{SwapStatusEvent, SwapStatusStreamer}; use super::swap_v2_common::*; use super::{swap_v2_topic, LockedAmount, LockedAmountInfo, SavedTradeFee, SwapsContext, NEGOTIATE_SEND_INTERVAL, NEGOTIATION_TIMEOUT_SEC}; @@ -20,6 +21,7 @@ use crypto::privkey::SerializableSecp256k1Keypair; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::EventStreamer; use mm2_libp2p::Secp256k1PubkeySerialize; use mm2_number::MmNumber; use mm2_state_machine::prelude::*; @@ -46,7 +48,7 @@ cfg_wasm32!( #[allow(unused_imports)] use prost::Message; /// Negotiation data representation to be stored in DB. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct StoredNegotiationData { taker_payment_locktime: u64, taker_funding_locktime: u64, @@ -58,7 +60,7 @@ pub struct StoredNegotiationData { } /// Represents events produced by maker swap states. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(tag = "event_type", content = "event_data")] pub enum MakerSwapEvent { /// Swap has been successfully initialized. @@ -716,12 +718,17 @@ impl (), } + // Send a notification to the swap status streamer about a new event. + self.ctx + .event_stream_manager + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::MakerV2 { + uuid: self.uuid, + event: event.clone(), + }) + .ok(); } - fn on_kickstart_event( - &mut self, - event: <::DbRepr as StateMachineDbRepr>::Event, - ) { + fn on_kickstart_event(&mut self, event: MakerSwapEvent) { match event { MakerSwapEvent::Initialized { maker_payment_trade_fee, diff --git a/mm2src/mm2_main/src/lp_swap/swap_events.rs b/mm2src/mm2_main/src/lp_swap/swap_events.rs new file mode 100644 index 0000000000..b86811926f --- /dev/null +++ b/mm2src/mm2_main/src/lp_swap/swap_events.rs @@ -0,0 +1,51 @@ +use super::maker_swap::MakerSavedEvent; +use super::maker_swap_v2::MakerSwapEvent; +use super::taker_swap::TakerSavedEvent; +use super::taker_swap_v2::TakerSwapEvent; +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; + +use async_trait::async_trait; +use futures::channel::oneshot; +use futures::StreamExt; +use uuid::Uuid; + +pub struct SwapStatusStreamer; + +impl SwapStatusStreamer { + pub fn new() -> Self { Self } +} + +// FIXME: This is sending only the individual events as they come in and not the full swap status. +// swap_v2_rpcs::SwapRpcData has much more information than this stream. Consider sending that instead? +#[derive(Serialize)] +#[serde(tag = "swap_type", content = "swap_data")] +pub enum SwapStatusEvent { + MakerV1 { uuid: Uuid, event: MakerSavedEvent }, + TakerV1 { uuid: Uuid, event: TakerSavedEvent }, + MakerV2 { uuid: Uuid, event: MakerSwapEvent }, + TakerV2 { uuid: Uuid, event: TakerSwapEvent }, +} + +#[async_trait] +impl EventStreamer for SwapStatusStreamer { + type DataInType = SwapStatusEvent; + + fn streamer_id(&self) -> String { "SWAP_STATUS".to_string() } + + async fn handle( + self, + broadcaster: StreamingManager, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + while let Some(swap_data) = data_rx.next().await { + let event_data = serde_json::to_value(swap_data).expect("Serialization shouldn't fail."); + let event = Event::new(self.streamer_id(), event_data); + broadcaster.broadcast(event); + } + } +} diff --git a/mm2src/mm2_main/src/lp_swap/swap_v2_common.rs b/mm2src/mm2_main/src/lp_swap/swap_v2_common.rs index ec87e9b79b..c4de1a17be 100644 --- a/mm2src/mm2_main/src/lp_swap/swap_v2_common.rs +++ b/mm2src/mm2_main/src/lp_swap/swap_v2_common.rs @@ -37,7 +37,7 @@ pub struct ActiveSwapV2Info { } /// DB representation of tx preimage with signature -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct StoredTxPreimage { pub preimage: BytesJson, pub signature: BytesJson, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap.rs b/mm2src/mm2_main/src/lp_swap/taker_swap.rs index 4e087ad527..b83e6fcc9d 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap.rs @@ -12,6 +12,7 @@ use super::{broadcast_my_swap_status, broadcast_swap_message, broadcast_swap_msg SwapsContext, TransactionIdentifier, INCLUDE_REFUND_FEE, NO_REFUND_FEE, WAIT_CONFIRM_INTERVAL_SEC}; use crate::mm2::lp_network::subscribe_to_topic; use crate::mm2::lp_ordermatch::TakerOrderBuilder; +use crate::mm2::lp_swap::swap_events::{SwapStatusEvent, SwapStatusStreamer}; use crate::mm2::lp_swap::swap_v2_common::mark_swap_as_finished; use crate::mm2::lp_swap::taker_restart::get_command_based_on_maker_or_watcher_activity; use crate::mm2::lp_swap::{broadcast_p2p_tx_msg, broadcast_swap_msg_every_delayed, tx_helper_topic, @@ -30,6 +31,7 @@ use http::Response; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; use mm2_rpc::data::legacy::{MatchBy, OrderConfirmationsSettings, TakerAction}; use parking_lot::Mutex as PaMutex; @@ -149,7 +151,7 @@ async fn save_my_taker_swap_event(ctx: &MmArc, swap: &TakerSwap, event: TakerSav } } -#[derive(Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct TakerSavedEvent { pub timestamp: u64, pub event: TakerSwapEvent, @@ -459,9 +461,16 @@ pub async fn run_taker_swap(swap: RunTakerSwapInput, ctx: MmArc) { event: event.clone(), }; - save_my_taker_swap_event(&ctx, &running_swap, to_save) + save_my_taker_swap_event(&ctx, &running_swap, to_save.clone()) .await .expect("!save_my_taker_swap_event"); + // Send a notification to the swap status streamer about a new event. + ctx.event_stream_manager + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::TakerV1 { + uuid: running_swap.uuid, + event: to_save, + }) + .ok(); if event.should_ban_maker() { ban_pubkey_on_failed_swap( &ctx, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs index 256503350b..9611be533a 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs @@ -1,3 +1,4 @@ +use super::swap_events::{SwapStatusEvent, SwapStatusStreamer}; use super::swap_v2_common::*; use super::{LockedAmount, LockedAmountInfo, SavedTradeFee, SwapsContext, TakerSwapPreparedParams, NEGOTIATE_SEND_INTERVAL, NEGOTIATION_TIMEOUT_SEC}; @@ -21,6 +22,7 @@ use crypto::privkey::SerializableSecp256k1Keypair; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::EventStreamer; use mm2_libp2p::Secp256k1PubkeySerialize; use mm2_number::MmNumber; use mm2_state_machine::prelude::*; @@ -47,7 +49,7 @@ cfg_wasm32!( #[allow(unused_imports)] use prost::Message; /// Negotiation data representation to be stored in DB. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct StoredNegotiationData { maker_payment_locktime: u64, maker_secret_hash: BytesJson, @@ -59,7 +61,7 @@ pub struct StoredNegotiationData { } /// Represents events produced by taker swap states. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(tag = "event_type", content = "event_data")] pub enum TakerSwapEvent { /// Swap has been successfully initialized. @@ -836,12 +838,17 @@ impl (), } + // Send a notification to the swap status streamer about a new event. + self.ctx + .event_stream_manager + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::TakerV2 { + uuid: self.uuid, + event: event.clone(), + }) + .ok(); } - fn on_kickstart_event( - &mut self, - event: <::DbRepr as StateMachineDbRepr>::Event, - ) { + fn on_kickstart_event(&mut self, event: TakerSwapEvent) { match event { TakerSwapEvent::Initialized { taker_payment_fee, .. } | TakerSwapEvent::Negotiated { taker_payment_fee, .. } => { diff --git a/mm2src/mm2_main/src/rpc.rs b/mm2src/mm2_main/src/rpc.rs index 8fca9eebca..7350cdacff 100644 --- a/mm2src/mm2_main/src/rpc.rs +++ b/mm2src/mm2_main/src/rpc.rs @@ -50,7 +50,8 @@ mod dispatcher_legacy; #[path = "rpc/lp_commands/lp_commands_legacy.rs"] pub mod lp_commands_legacy; #[path = "rpc/rate_limiter.rs"] mod rate_limiter; -#[path = "rpc/streaming_activations/mod.rs"] mod streaming_activations; +#[path = "rpc/streaming_activations/mod.rs"] +mod streaming_activations; /// Lists the RPC method not requiring the "userpass" authentication. /// None is also public to skip auth and display proper error in case of method is missing diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index af9f328fa4..5427494098 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -330,6 +330,7 @@ async fn rpc_streaming_dispatcher( ctx: MmArc, streaming_request: String, ) -> DispatcherResult>> { + // FIXME: Unify disable requests (they all run the same code based on streamer id, the routes make no sense). match streaming_request.as_str() { "balance::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_balance).await, "balance::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_balance).await, @@ -339,8 +340,8 @@ async fn rpc_streaming_dispatcher( "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat).await, "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation).await, "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation).await, - // "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, - // "swap_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_swap_status).await, + "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, + "swap_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_swap_status).await, // "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, // "order_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_order_status).await, _ => MmError::err(DispatcherError::NoSuchMethod), diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index 3431397fb6..ff715db7d0 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -35,8 +35,8 @@ pub async fn enable_heartbeat( ctx: MmArc, req: EnableHeartbeatRequest, ) -> MmResult { - let heartbeat_streamer = HeartbeatEvent::try_new(req.config) - .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}")))?; + let heartbeat_streamer = + HeartbeatEvent::try_new(req.config).map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}")))?; ctx.event_stream_manager .add(req.client_id, heartbeat_streamer, ctx.spawner()) .await diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index 7d96884081..5e6208e52b 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -3,7 +3,7 @@ mod fee_estimation; mod heartbeat; mod network; // mod orders; -// mod swaps; +mod swaps; // Re-exports pub use balance::*; @@ -11,7 +11,7 @@ pub use fee_estimation::*; pub use heartbeat::*; pub use network::*; // pub use orders::*; -// pub use swaps::*; +pub use swaps::*; #[derive(Serialize)] /// The success/ok response for any event streaming activation request. diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs new file mode 100644 index 0000000000..0dafc1c5b4 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs @@ -0,0 +1,51 @@ +//! RPC activation and deactivation of the swap status streamer. +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use crate::mm2::lp_swap::swap_events::SwapStatusStreamer; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use common::HttpStatusCode; +use http::StatusCode; + +#[derive(Deserialize)] +pub struct EnableSwapStatusStreamingRequest { + pub client_id: u64, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum SwapStatusStreamingRequestError { + EnableError(String), + DisableError(String), +} + +impl HttpStatusCode for SwapStatusStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + SwapStatusStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + SwapStatusStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + } + } +} + +pub async fn enable_swap_status( + ctx: MmArc, + req: EnableSwapStatusStreamingRequest, +) -> MmResult { + let swap_status_streamer = SwapStatusStreamer::new(); + ctx.event_stream_manager + .add(req.client_id, swap_status_streamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| SwapStatusStreamingRequestError::EnableError(format!("{e:?}"))) +} + +pub async fn disable_swap_status( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| SwapStatusStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) +} From 6fad6ab24ab4375422329d02d641621b9d91ab32 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 26 Jul 2024 14:50:58 +0200 Subject: [PATCH 060/128] add order status streaming --- mm2src/mm2_main/src/lp_ordermatch.rs | 36 +++++++++++++ .../src/lp_ordermatch/order_events.rs | 45 ++++++++++++++++ .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 4 +- .../src/rpc/streaming_activations/mod.rs | 4 +- .../src/rpc/streaming_activations/orders.rs | 51 +++++++++++++++++++ 5 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 mm2src/mm2_main/src/lp_ordermatch/order_events.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/orders.rs diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index 86e2fb0491..fdfa27131a 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -42,6 +42,7 @@ use http::Response; use keys::{AddressFormat, KeyPair}; use mm2_core::mm_ctx::{from_ctx, MmArc, MmWeak}; use mm2_err_handle::prelude::*; +use mm2_event_stream::EventStreamer; use mm2_libp2p::{decode_signed, encode_and_sign, encode_message, pub_sub_topic, PublicKey, TopicHash, TopicPrefix, TOPIC_SEPARATOR}; use mm2_metrics::mm_gauge; @@ -54,6 +55,7 @@ use my_orders_storage::{delete_my_maker_order, delete_my_taker_order, save_maker save_my_new_maker_order, save_my_new_taker_order, MyActiveOrders, MyOrdersFilteringHistory, MyOrdersHistory, MyOrdersStorage}; use num_traits::identities::Zero; +use order_events::{OrderStatusEvent, OrderStatusStreamer}; use parking_lot::Mutex as PaMutex; use rpc::v1::types::H256 as H256Json; use serde_json::{self as json, Value as Json}; @@ -103,6 +105,8 @@ pub use lp_bot::{start_simple_market_maker_bot, stop_simple_market_maker_bot, St #[path = "lp_ordermatch/my_orders_storage.rs"] mod my_orders_storage; #[path = "lp_ordermatch/new_protocol.rs"] mod new_protocol; +#[path = "lp_ordermatch/order_events.rs"] +pub(crate) mod order_events; #[path = "lp_ordermatch/order_requests_tracker.rs"] mod order_requests_tracker; #[path = "lp_ordermatch/orderbook_depth.rs"] mod orderbook_depth; @@ -3561,6 +3565,14 @@ async fn process_maker_reserved(ctx: MmArc, from_pubkey: H256Json, reserved_msg: connected: None, last_updated: now_ms(), }; + + ctx.event_stream_manager + .send( + &OrderStatusStreamer.streamer_id(), + OrderStatusEvent::TakerMatch(taker_match.clone()), + ) + .ok(); + my_order .matches .insert(taker_match.reserved.maker_order_uuid, taker_match); @@ -3609,6 +3621,14 @@ async fn process_maker_connected(ctx: MmArc, from_pubkey: PublicKey, connected: error!("Connected message sender pubkey != reserved message sender pubkey"); return; } + + ctx.event_stream_manager + .send( + &OrderStatusStreamer.streamer_id(), + OrderStatusEvent::TakerConnected(order_match.clone()), + ) + .ok(); + // alice lp_connected_alice( ctx.clone(), @@ -3715,6 +3735,14 @@ async fn process_taker_request(ctx: MmArc, from_pubkey: H256Json, taker_request: connected: None, last_updated: now_ms(), }; + + ctx.event_stream_manager + .send( + &OrderStatusStreamer.streamer_id(), + OrderStatusEvent::MakerMatch(maker_match.clone()), + ) + .ok(); + order.matches.insert(maker_match.request.uuid, maker_match); storage .update_active_maker_order(&order) @@ -3779,6 +3807,14 @@ async fn process_taker_connect(ctx: MmArc, sender_pubkey: PublicKey, connect_msg order_match.connect = Some(connect_msg); order_match.connected = Some(connected.clone()); let order_match = order_match.clone(); + + ctx.event_stream_manager + .send( + &OrderStatusStreamer.streamer_id(), + OrderStatusEvent::MakerConnected(order_match.clone()), + ) + .ok(); + my_order.started_swaps.push(order_match.request.uuid); lp_connect_start_bob(ctx.clone(), order_match, my_order.clone(), sender_pubkey); let topic = my_order.orderbook_topic(); diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs new file mode 100644 index 0000000000..2f8c11db47 --- /dev/null +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -0,0 +1,45 @@ +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; +use super::{MakerMatch, TakerMatch}; + +use async_trait::async_trait; +use futures::channel::oneshot; +use futures::StreamExt; + +pub struct OrderStatusStreamer; + +impl OrderStatusStreamer { + pub fn new() -> Self { Self } +} + +#[derive(Serialize)] +#[serde(tag = "order_type", content = "order_data")] +pub(super) enum OrderStatusEvent { + MakerMatch(MakerMatch), + TakerMatch(TakerMatch), + MakerConnected(MakerMatch), + TakerConnected(TakerMatch), +} + +#[async_trait] +impl EventStreamer for OrderStatusStreamer { + type DataInType = (); + + fn streamer_id(&self) -> String { "ORDER_STATUS".to_string() } + + async fn handle( + self, + broadcaster: StreamingManager, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + while let Some(swap_data) = data_rx.next().await { + let event_data = serde_json::to_value(swap_data).expect("Serialization shouldn't fail."); + let event = Event::new(self.streamer_id(), event_data); + broadcaster.broadcast(event); + } + } +} diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 5427494098..e218dbff2d 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -342,8 +342,8 @@ async fn rpc_streaming_dispatcher( "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation).await, "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, "swap_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_swap_status).await, - // "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, - // "order_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_order_status).await, + "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, + "order_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_order_status).await, _ => MmError::err(DispatcherError::NoSuchMethod), } } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index 5e6208e52b..63444278a9 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -2,7 +2,7 @@ mod balance; mod fee_estimation; mod heartbeat; mod network; -// mod orders; +mod orders; mod swaps; // Re-exports @@ -10,7 +10,7 @@ pub use balance::*; pub use fee_estimation::*; pub use heartbeat::*; pub use network::*; -// pub use orders::*; +pub use orders::*; pub use swaps::*; #[derive(Serialize)] diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs new file mode 100644 index 0000000000..05c4235ec0 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs @@ -0,0 +1,51 @@ +//! RPC activation and deactivation of the order status streamer. +use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use crate::mm2::lp_ordermatch::order_events::OrderStatusStreamer; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use common::HttpStatusCode; +use http::StatusCode; + +#[derive(Deserialize)] +pub struct EnableOrderStatusStreamingRequest { + pub client_id: u64, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum OrderStatusStreamingRequestError { + EnableError(String), + DisableError(String), +} + +impl HttpStatusCode for OrderStatusStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + OrderStatusStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + OrderStatusStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, + } + } +} + +pub async fn enable_order_status( + ctx: MmArc, + req: EnableOrderStatusStreamingRequest, +) -> MmResult { + let order_status_streamer = OrderStatusStreamer::new(); + ctx.event_stream_manager + .add(req.client_id, order_status_streamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| OrderStatusStreamingRequestError::EnableError(format!("{e:?}"))) +} + +pub async fn disable_order_status( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| OrderStatusStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) +} From b9d1218c9e8b4392188149b7434f85947f69b4d9 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 16:14:28 +0200 Subject: [PATCH 061/128] breaking: abandon event_stream_configuration config access_control_allow_origin config moved from: 'event_streaming_configuration' : { 'access_control_allow_origin': '*', ..other_params } to 'access_control_allow_origin': '*' i.e. moved one scope out in the mm2.json config --- mm2src/coins/lp_coins.rs | 2 - mm2src/coins/streaming_events_config.rs | 36 -------------- .../src/bch_with_tokens_activation.rs | 8 ---- .../src/eth_with_token_activation.rs | 8 ---- .../src/platform_coin_with_tokens.rs | 10 ---- .../src/solana_with_tokens_activation.rs | 8 ---- .../src/tendermint_with_assets_activation.rs | 8 ---- mm2src/mm2_core/src/mm_ctx.rs | 12 +---- mm2src/mm2_event_stream/src/configuration.rs | 48 ------------------- mm2src/mm2_event_stream/src/lib.rs | 2 - mm2src/mm2_main/src/lp_native_dex.rs | 6 +-- mm2src/mm2_net/src/sse_handler.rs | 7 ++- 12 files changed, 7 insertions(+), 148 deletions(-) delete mode 100644 mm2src/coins/streaming_events_config.rs delete mode 100644 mm2src/mm2_event_stream/src/configuration.rs diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 98e53a7b14..fe5d562168 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -237,8 +237,6 @@ use coin_errors::{MyAddressError, ValidatePaymentError, ValidatePaymentFut, Vali #[cfg(test)] pub mod coins_tests; -mod streaming_events_config; - pub mod eth; use eth::GetValidEthWithdrawAddError; use eth::{eth_coin_from_conf_and_request, get_eth_address, EthCoin, EthGasDetailsErr, EthTxFeeDetails, diff --git a/mm2src/coins/streaming_events_config.rs b/mm2src/coins/streaming_events_config.rs deleted file mode 100644 index abecb1b607..0000000000 --- a/mm2src/coins/streaming_events_config.rs +++ /dev/null @@ -1,36 +0,0 @@ -#![allow(dead_code)] -// FIXME: Get rid of this. - -use serde::Deserialize; -use serde_json::Value as Json; -use std::collections::HashMap; - -#[derive(Debug, Deserialize)] -pub(crate) struct BalanceEventConfig { - /// A map of coin tickers to their balance streaming configuration. - /// - /// The configuration doesn't have a specific structure at this point, every coin - /// has its own configuration structure. - coins: HashMap, -} - -impl BalanceEventConfig { - /// Returns the coin's configuration as a JSON object if it exists. - /// - /// Note that the `ticker` is looked up in a case-insensitive manner. - pub(crate) fn find_coin(&self, ticker: &str) -> Option { - // FIXME: Is this OK? are coin tickers like an identifier for the coin? Could two coins have the same ticker? - let ticker = ticker.to_lowercase(); - self.coins - .iter() - .find(|(key, _)| key.to_lowercase() == ticker) - .map(|(_, value)| value.clone()) - } -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -/// An empty configuration struct. -/// -/// It's used mainly to make sure there is no config provided. -pub(crate) struct EmptySubConfig {} diff --git a/mm2src/coins_activation/src/bch_with_tokens_activation.rs b/mm2src/coins_activation/src/bch_with_tokens_activation.rs index ebda8efcba..df05578354 100644 --- a/mm2src/coins_activation/src/bch_with_tokens_activation.rs +++ b/mm2src/coins_activation/src/bch_with_tokens_activation.rs @@ -19,7 +19,6 @@ use common::{drop_mutability, true_f}; use crypto::CryptoCtxError; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde_derive::{Deserialize, Serialize}; @@ -353,13 +352,6 @@ impl PlatformCoinWithTokensActivationOps for BchCoin { self.spawner().spawn_with_settings(fut, settings); } - async fn handle_balance_streaming( - &self, - _config: &EventStreamConfiguration, - ) -> Result<(), MmError> { - Ok(()) - } - fn rpc_task_manager( _activation_ctx: &CoinsActivationContext, ) -> &InitPlatformCoinWithTokensTaskManagerShared { diff --git a/mm2src/coins_activation/src/eth_with_token_activation.rs b/mm2src/coins_activation/src/eth_with_token_activation.rs index 296cfcfd73..6424005138 100644 --- a/mm2src/coins_activation/src/eth_with_token_activation.rs +++ b/mm2src/coins_activation/src/eth_with_token_activation.rs @@ -25,7 +25,6 @@ use common::{drop_mutability, true_f}; use crypto::HwRpcError; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamConfiguration; #[cfg(target_arch = "wasm32")] use mm2_metamask::MetamaskRpcError; use mm2_number::BigDecimal; @@ -437,13 +436,6 @@ impl PlatformCoinWithTokensActivationOps for EthCoin { ) { } - async fn handle_balance_streaming( - &self, - _config: &EventStreamConfiguration, - ) -> Result<(), MmError> { - Ok(()) - } - fn rpc_task_manager( activation_ctx: &CoinsActivationContext, ) -> &InitPlatformCoinWithTokensTaskManagerShared { diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index 0d031e7808..83566123db 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -13,7 +13,6 @@ use crypto::CryptoCtxError; use derive_more::Display; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError, RpcTaskUserActionRequest}; @@ -214,11 +213,6 @@ pub trait PlatformCoinWithTokensActivationOps: Into + Clone + Send + initial_balance: Option, ); - async fn handle_balance_streaming( - &self, - config: &EventStreamConfiguration, - ) -> Result<(), MmError>; - fn rpc_task_manager(activation_ctx: &CoinsActivationContext) -> &InitPlatformCoinWithTokensTaskManagerShared where EnablePlatformCoinWithTokensError: From; @@ -481,10 +475,6 @@ where ); } - platform_coin - .handle_balance_streaming(&ctx.event_stream_configuration) - .await?; - let coins_ctx = CoinsContext::from_ctx(&ctx).unwrap(); coins_ctx .add_platform_with_tokens(platform_coin.into(), mm_tokens, nft_global) diff --git a/mm2src/coins_activation/src/solana_with_tokens_activation.rs b/mm2src/coins_activation/src/solana_with_tokens_activation.rs index f411995779..697ef10eef 100644 --- a/mm2src/coins_activation/src/solana_with_tokens_activation.rs +++ b/mm2src/coins_activation/src/solana_with_tokens_activation.rs @@ -22,7 +22,6 @@ use crypto::CryptoCtxError; use futures::future::try_join_all; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde_derive::{Deserialize, Serialize}; @@ -312,13 +311,6 @@ impl PlatformCoinWithTokensActivationOps for SolanaCoin { ) { } - async fn handle_balance_streaming( - &self, - _config: &EventStreamConfiguration, - ) -> Result<(), MmError> { - Ok(()) - } - fn rpc_task_manager( _activation_ctx: &CoinsActivationContext, ) -> &InitPlatformCoinWithTokensTaskManagerShared { diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 34bf3bdfed..6ea17a3bfd 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -21,7 +21,6 @@ use common::executor::{AbortSettings, SpawnAbortable}; use common::{true_f, Future01CompatExt}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamConfiguration; use mm2_number::BigDecimal; use rpc_task::RpcTaskHandleShared; use serde::{Deserialize, Deserializer, Serialize}; @@ -368,13 +367,6 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { self.spawner().spawn_with_settings(fut, settings); } - async fn handle_balance_streaming( - &self, - _config: &EventStreamConfiguration, - ) -> Result<(), MmError> { - Ok(()) - } - fn rpc_task_manager( _activation_ctx: &CoinsActivationContext, ) -> &InitPlatformCoinWithTokensTaskManagerShared { diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 9aeef80721..63bb6c3cb2 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::{EventStreamConfiguration, StreamingManager}; +use mm2_event_stream::StreamingManager; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; @@ -77,8 +77,6 @@ pub struct MmCtx { pub rpc_started: Constructible, /// Data transfer bridge between server and client where server (which is the mm2 runtime) initiates the request. pub(crate) data_asker: DataAsker, - /// Configuration of event streaming used for SSE. - pub event_stream_configuration: EventStreamConfiguration, /// A manager for the event streaming system. To be used to start/stop/communicate with event streamers. pub event_stream_manager: StreamingManager, /// True if the MarketMaker instance needs to stop. @@ -153,7 +151,6 @@ impl MmCtx { initialized: Constructible::default(), rpc_started: Constructible::default(), data_asker: DataAsker::default(), - event_stream_configuration: Default::default(), event_stream_manager: Default::default(), stop: Constructible::default(), ffi_handle: Constructible::default(), @@ -715,13 +712,6 @@ impl MmCtxBuilder { if let Some(conf) = self.conf { ctx.conf = conf; - - let event_stream_configuration = &ctx.conf["event_stream_configuration"]; - if !event_stream_configuration.is_null() { - // FIXME: Consider validating all the fields in the event streaming configuration here. - ctx.event_stream_configuration = json::from_value(event_stream_configuration.clone()) - .expect("Invalid json value in 'event_stream_configuration'."); - } } #[cfg(target_arch = "wasm32")] diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs deleted file mode 100644 index b34707ecb9..0000000000 --- a/mm2src/mm2_event_stream/src/configuration.rs +++ /dev/null @@ -1,48 +0,0 @@ -use std::collections::HashMap; - -use serde::Deserialize; -use serde_json::Value as Json; - -#[cfg(target_arch = "wasm32")] use std::path::PathBuf; - -#[cfg(target_arch = "wasm32")] -const DEFAULT_WORKER_PATH: &str = "event_streaming_worker.js"; - -/// Configuration for event streaming -#[derive(Deserialize)] -pub struct EventStreamConfiguration { - /// The value to set for the `Access-Control-Allow-Origin` header. - #[serde(default)] - pub access_control_allow_origin: String, - #[serde(default)] - active_events: HashMap, - /// The path to the worker script for event streaming. - #[cfg(target_arch = "wasm32")] - #[serde(default = "default_worker_path")] - pub worker_path: PathBuf, -} - -#[cfg(target_arch = "wasm32")] -#[inline] -fn default_worker_path() -> PathBuf { PathBuf::from(DEFAULT_WORKER_PATH) } - -impl Default for EventStreamConfiguration { - fn default() -> Self { - Self { - access_control_allow_origin: String::from("*"), - active_events: Default::default(), - #[cfg(target_arch = "wasm32")] - worker_path: default_worker_path(), - } - } -} - -impl EventStreamConfiguration { - /// Retrieves the configuration for a specific event by its name. - #[inline] - pub fn get_event(&self, event_name: &str) -> Option { self.active_events.get(event_name).cloned() } - - /// Gets the total number of active events in the configuration. - #[inline] - pub fn total_active_events(&self) -> usize { self.active_events.len() } -} diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 7a85762119..a2b14860b3 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,11 +1,9 @@ -pub mod configuration; mod controller; pub mod event; pub mod manager; pub mod streamer; // Re-export important types. -pub use configuration::EventStreamConfiguration; pub use event::Event; pub use manager::{StreamingManager, StreamingManagerError}; pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index e2c421ff5b..d6773499fa 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -423,11 +423,7 @@ fn migrate_db(ctx: &MmArc) -> MmInitResult<()> { fn migration_1(_ctx: &MmArc) {} #[cfg(target_arch = "wasm32")] -fn init_wasm_event_streaming(ctx: &MmArc) { - if ctx.event_stream_configuration.total_active_events() != 0 { - ctx.spawner().spawn(handle_worker_stream(ctx.clone())); - } -} +fn init_wasm_event_streaming(ctx: &MmArc) { ctx.spawner().spawn(handle_worker_stream(ctx.clone())); } pub async fn lp_init_continue(ctx: MmArc) -> MmInitResult<()> { init_ordermatch_context(&ctx)?; diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index f5f6579bc5..62380cc07e 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -14,7 +14,10 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result return handle_internal_error(err).await, }; - let config = &ctx.event_stream_configuration; + let access_control = ctx.conf["access_control_allow_origin"] + .as_str() + .unwrap_or("*") + .to_string(); let Some(Ok(client_id)) = request.uri().query().and_then(|query| { query .split('&') @@ -53,7 +56,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Mon, 29 Jul 2024 16:30:08 +0200 Subject: [PATCH 062/128] ignore utxo scripthash subscriptions fixme for now moved to a TODO, we need #1966 first to start working on it, otherwise it's gonna look like this: https://github.com/KomodoPlatform/komodo-defi-framework/pull/2172/commits/8972822af21587b02e2285857f8f4d4d6ba280cc not removing the ScripthashNotificationSender traces entierly so not to have a lot of merge conflicts with #1966, but it's now disabled (provided always as None). --- mm2src/coins/utxo/rpc_clients.rs | 4 +++- .../utxo/utxo_builder/utxo_coin_builder.rs | 18 ++---------------- .../mm2_main/src/lp_ordermatch/order_events.rs | 2 +- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index f2d14395c8..6ef3033607 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -2605,9 +2605,11 @@ async fn electrum_process_json( }, }; + // TODO: Use the streaming manager instead to send the notification. + // This is disabled (set as None from the provider) for now untill the refactor in #1966 + // is available. Leaving it as is for now to avoid lots of conflicts. if let Some(sender) = scripthash_notification_sender { debug!("Sending scripthash message"); - // FIXME: Nobody is listening to this channel. Use the streaming manager. if let Err(e) = sender.unbounded_send(ScripthashNotification::Triggered(scripthash.to_string())) { error!("Failed sending scripthash message. {e}"); diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index bac10fb2cf..62f12103e8 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -265,18 +265,11 @@ where let my_script_pubkey = output_script(&my_address).map(|script| script.to_bytes())?; - let (scripthash_notification_sender, _) = { - let (sender, receiver) = futures::channel::mpsc::unbounded(); - (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) - }; - // Create an abortable system linked to the `MmCtx` so if the context is stopped via `MmArc::stop`, // all spawned futures related to this `UTXO` coin will be aborted as well. let abortable_system: AbortableQueue = builder.ctx().abortable_system.create_subsystem()?; - let rpc_client = builder - .rpc_client(scripthash_notification_sender, abortable_system.create_subsystem()?) - .await?; + let rpc_client = builder.rpc_client(None, abortable_system.create_subsystem()?).await?; let tx_fee = builder.tx_fee(&rpc_client).await?; let decimals = builder.decimals(&rpc_client).await?; let dust_amount = builder.dust_amount(); @@ -349,18 +342,11 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { address_format, }; - let (scripthash_notification_sender, _) = { - let (sender, receiver) = futures::channel::mpsc::unbounded(); - (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) - }; - // Create an abortable system linked to the `MmCtx` so if the context is stopped via `MmArc::stop`, // all spawned futures related to this `UTXO` coin will be aborted as well. let abortable_system: AbortableQueue = self.ctx().abortable_system.create_subsystem()?; - let rpc_client = self - .rpc_client(scripthash_notification_sender, abortable_system.create_subsystem()?) - .await?; + let rpc_client = self.rpc_client(None, abortable_system.create_subsystem()?).await?; let tx_fee = self.tx_fee(&rpc_client).await?; let decimals = self.decimals(&rpc_client).await?; let dust_amount = self.dust_amount(); diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 2f8c11db47..b4c29906fb 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -1,5 +1,5 @@ -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; use super::{MakerMatch, TakerMatch}; +use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; use async_trait::async_trait; use futures::channel::oneshot; From f54f502081726631d2eed9b2f185bbc9b6ef4a1f Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 17:20:23 +0200 Subject: [PATCH 063/128] unify all disable streaming endpoints they all share the same request and response and don't depend on the path --- .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 8 +-- .../src/rpc/streaming_activations/balance.rs | 14 +----- .../src/rpc/streaming_activations/disable.rs | 50 +++++++++++++++++++ .../streaming_activations/fee_estimation.rs | 14 +----- .../rpc/streaming_activations/heartbeat.rs | 18 +------ .../src/rpc/streaming_activations/mod.rs | 21 ++------ .../src/rpc/streaming_activations/network.rs | 20 +------- .../src/rpc/streaming_activations/orders.rs | 18 +------ .../src/rpc/streaming_activations/swaps.rs | 20 +------- 9 files changed, 64 insertions(+), 119 deletions(-) create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/disable.rs diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index e218dbff2d..45777ea308 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -330,20 +330,14 @@ async fn rpc_streaming_dispatcher( ctx: MmArc, streaming_request: String, ) -> DispatcherResult>> { - // FIXME: Unify disable requests (they all run the same code based on streamer id, the routes make no sense). match streaming_request.as_str() { "balance::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_balance).await, - "balance::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_balance).await, "network::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_network).await, - "network::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_network).await, "heartbeat::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_heartbeat).await, - "heartbeat::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_heartbeat).await, "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation).await, - "fee_estimator::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_fee_estimation).await, "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, - "swap_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_swap_status).await, "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, - "order_status::disable" => handle_mmrpc(ctx, request, streaming_activations::disable_order_status).await, + "disable" => handle_mmrpc(ctx, request, streaming_activations::disable_streamer).await, _ => MmError::err(DispatcherError::NoSuchMethod), } } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 8ec8b9aaad..3f5f0546d7 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different balance event streamers. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::{EnableStreamingResponse}; use coins::eth::eth_balance_events::EthBalanceEventStreamer; use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; @@ -24,7 +24,6 @@ pub struct EnableBalanceStreamingRequest { #[serde(tag = "error_type", content = "error_data")] pub enum BalanceStreamingRequestError { EnableError(String), - DisableError(String), CoinNotFound, CoinNotSupported, Internal(String), @@ -34,7 +33,6 @@ impl HttpStatusCode for BalanceStreamingRequestError { fn status_code(&self) -> StatusCode { match self { BalanceStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - BalanceStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, BalanceStreamingRequestError::CoinNotFound => StatusCode::NOT_FOUND, BalanceStreamingRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, BalanceStreamingRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, @@ -122,13 +120,3 @@ pub async fn enable_balance( .map(EnableStreamingResponse::new) .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}"))) } - -pub async fn disable_balance( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| BalanceStreamingRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs b/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs new file mode 100644 index 0000000000..ab1155378e --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs @@ -0,0 +1,50 @@ +//! The module for handling any event streaming deactivation requests. +//! +//! All event streamers are deactivated using the streamer ID only. + +use common::HttpStatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use http::StatusCode; + +#[derive(Deserialize)] +/// The request used for any event streaming deactivation. +pub struct DisableStreamingRequest { + pub client_id: u64, + pub streamer_id: String, +} + +#[derive(Serialize)] +/// The success/ok response for any event streaming deactivation request. +pub struct DisableStreamingResponse { + result: &'static str, +} + +impl DisableStreamingResponse { + fn new() -> Self { Self { result: "Success" } } +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +/// The error response for any event streaming deactivation request. +pub enum DisableStreamingRequestError { + DisableError(String), +} + +impl HttpStatusCode for DisableStreamingRequestError { + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } +} + +/// Disables a streamer. +/// +/// This works for any streamer regarding of their type/usage. +pub async fn disable_streamer( + ctx: MmArc, + req: DisableStreamingRequest, +) -> MmResult { + ctx.event_stream_manager + .stop(req.client_id, &req.streamer_id) + .map_to_mm(|e| DisableStreamingRequestError::DisableError(format!("{e:?}")))?; + Ok(DisableStreamingResponse::new()) +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index 9a46a88c06..437c996938 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different fee estimation streamers. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::{EnableStreamingResponse}; use coins::eth::fee_estimation::eth_fee_events::EthFeeEventStreamer; use coins::{lp_coinfind, MmCoin, MmCoinEnum}; @@ -21,7 +21,6 @@ pub struct EnableFeeStreamingRequest { #[serde(tag = "error_type", content = "error_data")] pub enum FeeStreamingRequestError { EnableError(String), - DisableError(String), CoinNotFound, CoinNotSupported, Internal(String), @@ -31,7 +30,6 @@ impl HttpStatusCode for FeeStreamingRequestError { fn status_code(&self) -> StatusCode { match self { FeeStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - FeeStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, FeeStreamingRequestError::CoinNotFound => StatusCode::NOT_FOUND, FeeStreamingRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, FeeStreamingRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, @@ -61,13 +59,3 @@ pub async fn enable_fee_estimation( _ => Err(FeeStreamingRequestError::CoinNotSupported)?, } } - -pub async fn disable_fee_estimation( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| FeeStreamingRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index ff715db7d0..de997bd8da 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the heartbeats. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::{EnableStreamingResponse}; use crate::mm2::heartbeat_event::HeartbeatEvent; use common::HttpStatusCode; @@ -19,15 +19,11 @@ pub struct EnableHeartbeatRequest { #[serde(tag = "error_type", content = "error_data")] pub enum HeartbeatRequestError { EnableError(String), - DisableError(String), } impl HttpStatusCode for HeartbeatRequestError { fn status_code(&self) -> StatusCode { - match self { - HeartbeatRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - HeartbeatRequestError::DisableError(_) => StatusCode::BAD_REQUEST, - } + StatusCode::BAD_REQUEST } } @@ -43,13 +39,3 @@ pub async fn enable_heartbeat( .map(EnableStreamingResponse::new) .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}"))) } - -pub async fn disable_heartbeat( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| HeartbeatRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index 63444278a9..e8629a9a65 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -1,4 +1,5 @@ mod balance; +mod disable; mod fee_estimation; mod heartbeat; mod network; @@ -7,6 +8,7 @@ mod swaps; // Re-exports pub use balance::*; +pub use disable::*; pub use fee_estimation::*; pub use heartbeat::*; pub use network::*; @@ -15,7 +17,7 @@ pub use swaps::*; #[derive(Serialize)] /// The success/ok response for any event streaming activation request. -/// Note that we don't have a unified EnableStreamingRequest. It is defined per event streaming activation. +/// Note that we don't have a unified request. It is rather defined per event streaming activation. pub struct EnableStreamingResponse { pub streamer_id: String, // FIXME: Consider returning the applied config here (might be different from the one the client requested). @@ -25,20 +27,3 @@ pub struct EnableStreamingResponse { impl EnableStreamingResponse { fn new(streamer_id: String) -> Self { Self { streamer_id } } } - -#[derive(Deserialize)] -/// The request used for any event streaming deactivation. -pub struct DisableStreamingRequest { - pub client_id: u64, - pub streamer_id: String, -} - -#[derive(Serialize)] -/// The success/ok response for any event streaming deactivation request. -pub struct DisableStreamingResponse { - result: &'static str, -} - -impl DisableStreamingResponse { - fn new() -> Self { Self { result: "Success" } } -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs index 8a95f5b76b..b32234ec1b 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the network event streamer. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::EnableStreamingResponse; use common::HttpStatusCode; use http::StatusCode; @@ -19,16 +19,10 @@ pub struct EnableNetworkStreamingRequest { #[serde(tag = "error_type", content = "error_data")] pub enum NetworkStreamingRequestError { EnableError(String), - DisableError(String), } impl HttpStatusCode for NetworkStreamingRequestError { - fn status_code(&self) -> StatusCode { - match self { - NetworkStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - NetworkStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, - } - } + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } } pub async fn enable_network( @@ -43,13 +37,3 @@ pub async fn enable_network( .map(EnableStreamingResponse::new) .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}"))) } - -pub async fn disable_network( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| NetworkStreamingRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs index 05c4235ec0..e7dfe3dd59 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation of the order status streamer. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::{EnableStreamingResponse}; use crate::mm2::lp_ordermatch::order_events::OrderStatusStreamer; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; @@ -16,15 +16,11 @@ pub struct EnableOrderStatusStreamingRequest { #[serde(tag = "error_type", content = "error_data")] pub enum OrderStatusStreamingRequestError { EnableError(String), - DisableError(String), } impl HttpStatusCode for OrderStatusStreamingRequestError { fn status_code(&self) -> StatusCode { - match self { - OrderStatusStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - OrderStatusStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, - } + StatusCode::BAD_REQUEST } } @@ -39,13 +35,3 @@ pub async fn enable_order_status( .map(EnableStreamingResponse::new) .map_to_mm(|e| OrderStatusStreamingRequestError::EnableError(format!("{e:?}"))) } - -pub async fn disable_order_status( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| OrderStatusStreamingRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs index 0dafc1c5b4..5a381166b0 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation of the swap status streamer. -use super::{DisableStreamingRequest, DisableStreamingResponse, EnableStreamingResponse}; +use super::EnableStreamingResponse; use crate::mm2::lp_swap::swap_events::SwapStatusStreamer; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; @@ -16,16 +16,10 @@ pub struct EnableSwapStatusStreamingRequest { #[serde(tag = "error_type", content = "error_data")] pub enum SwapStatusStreamingRequestError { EnableError(String), - DisableError(String), } impl HttpStatusCode for SwapStatusStreamingRequestError { - fn status_code(&self) -> StatusCode { - match self { - SwapStatusStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, - SwapStatusStreamingRequestError::DisableError(_) => StatusCode::BAD_REQUEST, - } - } + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } } pub async fn enable_swap_status( @@ -39,13 +33,3 @@ pub async fn enable_swap_status( .map(EnableStreamingResponse::new) .map_to_mm(|e| SwapStatusStreamingRequestError::EnableError(format!("{e:?}"))) } - -pub async fn disable_swap_status( - ctx: MmArc, - req: DisableStreamingRequest, -) -> MmResult { - ctx.event_stream_manager - .stop(req.client_id, &req.streamer_id) - .map_to_mm(|e| SwapStatusStreamingRequestError::DisableError(format!("{e:?}")))?; - Ok(DisableStreamingResponse::new()) -} From 481bf5df4ab9f9ba83a65d7a8c25c4cd233c76c0 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 17:21:24 +0200 Subject: [PATCH 064/128] remove a fixmes to show up in review if it's important :) --- mm2src/mm2_event_stream/src/streamer.rs | 3 --- mm2src/mm2_main/src/lp_swap/swap_events.rs | 2 -- mm2src/mm2_net/src/sse_handler.rs | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index d19f26220d..e5334a260e 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -48,9 +48,6 @@ where /// This method should not be overridden. async fn spawn( self, - // FIXME: Might be better to let the implementors store the spawner themselves and - // we can use `self.spawner()` here to get it. - // Also for AbortSettings, we can make this customizable with a default impl. spawner: WeakSpawner, broadcaster: StreamingManager, ) -> Result<(oneshot::Sender<()>, Option>>), String> { diff --git a/mm2src/mm2_main/src/lp_swap/swap_events.rs b/mm2src/mm2_main/src/lp_swap/swap_events.rs index b86811926f..624b2ef920 100644 --- a/mm2src/mm2_main/src/lp_swap/swap_events.rs +++ b/mm2src/mm2_main/src/lp_swap/swap_events.rs @@ -15,8 +15,6 @@ impl SwapStatusStreamer { pub fn new() -> Self { Self } } -// FIXME: This is sending only the individual events as they come in and not the full swap status. -// swap_v2_rpcs::SwapRpcData has much more information than this stream. Consider sending that instead? #[derive(Serialize)] #[serde(tag = "swap_type", content = "swap_data")] pub enum SwapStatusEvent { diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/sse_handler.rs index 62380cc07e..1b931142ef 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/sse_handler.rs @@ -31,7 +31,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Mon, 29 Jul 2024 18:02:44 +0200 Subject: [PATCH 065/128] broadcast all from the data asker since there is no such a streamer id (the id was made up on the spot in the data asker) the client who should receive this message won't get it (not any client in fact). it seems like we can't somehow know the client, so sending the request to all clients instead. --- mm2src/mm2_core/src/data_asker.rs | 4 ++-- mm2src/mm2_event_stream/src/manager.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index 21c5211047..9a0c64ef90 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -70,9 +70,9 @@ impl MmCtx { "data": data }); - // FIXME: Use broadcast_all here? + // FIXME: This event will be broadcasted to all the clients. We should only hand it to one client? self.event_stream_manager - .broadcast(Event::new(format!("{EVENT_NAME}:{data_type}"), input)); + .broadcast_all(Event::new(format!("{EVENT_NAME}:{data_type}"), input)); match receiver.timeout(timeout).await { Ok(Ok(response)) => match serde_json::from_value::(response) { diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 8c50efec16..39928c0d10 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -195,7 +195,7 @@ impl StreamingManager { } } - /// Forcefully broadcasts an event to all clients even if they are not listening for such an event. + /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. pub fn broadcast_all(&self, event: Event) { self.controller.read().unwrap().broadcast(event, None) } pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { From fb4e1eed0103d65934dfd9f68505eba9b945c59e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 18:15:19 +0200 Subject: [PATCH 066/128] use a wrapper around StreamingManager in EventStreamer --- mm2src/coins/eth/eth_balance_events.rs | 6 +++--- .../coins/eth/fee_estimation/eth_fee_events.rs | 4 ++-- .../tendermint/tendermint_balance_events.rs | 4 ++-- mm2src/coins/utxo/utxo_balance_events.rs | 4 ++-- mm2src/coins/z_coin/z_balance_streaming.rs | 4 ++-- mm2src/mm2_event_stream/src/lib.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 3 ++- mm2src/mm2_event_stream/src/streamer.rs | 17 ++++++++++++----- mm2src/mm2_main/src/heartbeat_event.rs | 4 ++-- .../mm2_main/src/lp_ordermatch/order_events.rs | 4 ++-- mm2src/mm2_main/src/lp_swap/swap_events.rs | 4 ++-- .../src/rpc/streaming_activations/balance.rs | 2 +- .../rpc/streaming_activations/fee_estimation.rs | 2 +- .../src/rpc/streaming_activations/heartbeat.rs | 6 ++---- .../src/rpc/streaming_activations/orders.rs | 6 ++---- mm2src/mm2_net/src/network_event.rs | 4 ++-- 16 files changed, 40 insertions(+), 36 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index e173d6c471..17ac8bc5aa 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -3,7 +3,7 @@ use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, BalanceError, CoinWithDerivationMethod}; use common::{executor::Timer, log, Future01CompatExt}; use mm2_err_handle::prelude::MmError; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use async_trait::async_trait; @@ -149,13 +149,13 @@ impl EventStreamer for EthBalanceEventStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - async fn start_polling(streamer_id: String, broadcaster: StreamingManager, coin: EthCoin, interval: f64) { + async fn start_polling(streamer_id: String, broadcaster: Broadcaster, coin: EthCoin, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, // our goal is to broadcast changed balances every x seconds. diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index badabf80dd..86001b833d 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -1,7 +1,7 @@ use super::eip1559::GasApiConfig; use crate::eth::EthCoin; use common::executor::Timer; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use async_trait::async_trait; use futures::channel::oneshot; @@ -54,7 +54,7 @@ impl EventStreamer for EthFeeEventStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { diff --git a/mm2src/coins/tendermint/tendermint_balance_events.rs b/mm2src/coins/tendermint/tendermint_balance_events.rs index abddf26bfc..078d0cadee 100644 --- a/mm2src/coins/tendermint/tendermint_balance_events.rs +++ b/mm2src/coins/tendermint/tendermint_balance_events.rs @@ -4,7 +4,7 @@ use futures::channel::oneshot; use futures_util::{SinkExt, StreamExt}; use jsonrpc_core::MethodCall; use jsonrpc_core::{Id as RpcId, Params as RpcParams, Value as RpcValue, Version as RpcVersion}; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; use std::collections::{HashMap, HashSet}; @@ -27,7 +27,7 @@ impl EventStreamer for TendermintBalanceEventStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 1036969369..dcd18563b5 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -3,7 +3,7 @@ use common::{executor::Timer, log, Future01CompatExt}; use futures::channel::oneshot; use futures::StreamExt; use keys::Address; -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; use std::collections::{BTreeMap, HashSet}; use super::{utxo_standard::UtxoStandardCoin, UtxoArc}; @@ -47,7 +47,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 63e7561110..5909b5cb42 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -8,7 +8,7 @@ use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; use std::sync::Arc; pub type ZBalanceEventSender = UnboundedSender<()>; @@ -30,7 +30,7 @@ impl EventStreamer for ZCoinBalanceEventStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput<()>, ) { diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index a2b14860b3..5137d0953c 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -6,4 +6,4 @@ pub mod streamer; // Re-export important types. pub use event::Event; pub use manager::{StreamingManager, StreamingManagerError}; -pub use streamer::{EventStreamer, NoDataIn, StreamHandlerInput}; +pub use streamer::{Broadcaster, EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 39928c0d10..61c1078f74 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,6 +2,7 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, RwLock}; +use crate::streamer::Broadcaster; use crate::{controller::Controller, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; @@ -115,7 +116,7 @@ impl StreamingManager { // Spawn a new streamer. let (shutdown, data_in) = streamer - .spawn(spawner, self.clone()) + .spawn(spawner, Broadcaster::new(self.clone())) .await .map_err(StreamingManagerError::SpawnError)?; let streamer_info = StreamerInfo::new(data_in, shutdown); diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index e5334a260e..0a74b70313 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -1,6 +1,6 @@ use std::any::{self, Any}; -use crate::StreamingManager; +use crate::{Event, StreamingManager}; use common::executor::{abortable_queue::WeakSpawner, AbortSettings, SpawnAbortable}; use common::log::{error, info}; @@ -33,9 +33,7 @@ where /// `data_rx` is a receiver that the streamer *could* use to receive data from the outside world. async fn handle( self, - // FIXME: Consider creating a new wrapper type around `StreamingManager` - // that won't expose any other method except `broadcast`. - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, ); @@ -49,7 +47,7 @@ where async fn spawn( self, spawner: WeakSpawner, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ) -> Result<(oneshot::Sender<()>, Option>>), String> { let streamer_id = self.streamer_id(); info!("Spawning event streamer: {streamer_id}"); @@ -104,3 +102,12 @@ where } } } + +/// A wrapper around `StreamingManager` to only expose the `broadcast` method. +pub struct Broadcaster(StreamingManager); + +impl Broadcaster { + pub fn new(inner: StreamingManager) -> Self { Self(inner) } + + pub fn broadcast(&self, event: Event) { self.0.broadcast(event); } +} diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index bccd9a876e..21175aafbf 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use common::executor::Timer; use futures::channel::oneshot; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use serde::Deserialize; use serde_json::Value as Json; @@ -40,7 +40,7 @@ impl EventStreamer for HeartbeatEvent { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index b4c29906fb..1796beae26 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -1,5 +1,5 @@ use super::{MakerMatch, TakerMatch}; -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; use async_trait::async_trait; use futures::channel::oneshot; @@ -28,7 +28,7 @@ impl EventStreamer for OrderStatusStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { diff --git a/mm2src/mm2_main/src/lp_swap/swap_events.rs b/mm2src/mm2_main/src/lp_swap/swap_events.rs index 624b2ef920..d5b2217ed1 100644 --- a/mm2src/mm2_main/src/lp_swap/swap_events.rs +++ b/mm2src/mm2_main/src/lp_swap/swap_events.rs @@ -2,7 +2,7 @@ use super::maker_swap::MakerSavedEvent; use super::maker_swap_v2::MakerSwapEvent; use super::taker_swap::TakerSavedEvent; use super::taker_swap_v2::TakerSwapEvent; -use mm2_event_stream::{Event, EventStreamer, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; use async_trait::async_trait; use futures::channel::oneshot; @@ -32,7 +32,7 @@ impl EventStreamer for SwapStatusStreamer { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 3f5f0546d7..b938bfe217 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different balance event streamers. -use super::{EnableStreamingResponse}; +use super::EnableStreamingResponse; use coins::eth::eth_balance_events::EthBalanceEventStreamer; use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index 437c996938..b9350739bb 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different fee estimation streamers. -use super::{EnableStreamingResponse}; +use super::EnableStreamingResponse; use coins::eth::fee_estimation::eth_fee_events::EthFeeEventStreamer; use coins::{lp_coinfind, MmCoin, MmCoinEnum}; diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index de997bd8da..c92909b5d1 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the heartbeats. -use super::{EnableStreamingResponse}; +use super::EnableStreamingResponse; use crate::mm2::heartbeat_event::HeartbeatEvent; use common::HttpStatusCode; @@ -22,9 +22,7 @@ pub enum HeartbeatRequestError { } impl HttpStatusCode for HeartbeatRequestError { - fn status_code(&self) -> StatusCode { - StatusCode::BAD_REQUEST - } + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } } pub async fn enable_heartbeat( diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs index e7dfe3dd59..38d9f74493 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation of the order status streamer. -use super::{EnableStreamingResponse}; +use super::EnableStreamingResponse; use crate::mm2::lp_ordermatch::order_events::OrderStatusStreamer; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; @@ -19,9 +19,7 @@ pub enum OrderStatusStreamingRequestError { } impl HttpStatusCode for OrderStatusStreamingRequestError { - fn status_code(&self) -> StatusCode { - StatusCode::BAD_REQUEST - } + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } } pub async fn enable_order_status( diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 1f45552507..6f3efcb023 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -1,7 +1,7 @@ use crate::p2p::P2PContext; use common::executor::Timer; use mm2_core::mm_ctx::MmArc; -use mm2_event_stream::{Event, EventStreamer, NoDataIn, StreamHandlerInput, StreamingManager}; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_libp2p::behaviours::atomicdex; use async_trait::async_trait; @@ -49,7 +49,7 @@ impl EventStreamer for NetworkEvent { async fn handle( self, - broadcaster: StreamingManager, + broadcaster: Broadcaster, ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { From d347e4ef7b1677c5c91b1c75a0a31b10cbd128ef Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 19:15:35 +0200 Subject: [PATCH 067/128] eth balance streaming: attach the address that caused balance fetch error in the streamed error message --- mm2src/coins/eth/eth_balance_events.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 17ac8bc5aa..1f3249b7e8 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -2,7 +2,6 @@ use super::EthCoin; use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, BalanceError, CoinWithDerivationMethod}; use common::{executor::Timer, log, Future01CompatExt}; -use mm2_err_handle::prelude::MmError; use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; @@ -52,10 +51,11 @@ struct BalanceData { balance: BigDecimal, } +#[derive(Serialize)] struct BalanceFetchError { ticker: String, address: String, - error: MmError, + error: BalanceError, } type BalanceResult = Result; @@ -111,7 +111,7 @@ async fn fetch_balance( .map_err(|error| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error, + error: error.into_inner(), })?, coin.decimals, ) @@ -122,7 +122,7 @@ async fn fetch_balance( .map_err(|error| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error, + error: error.into_inner(), })?, info.decimals, ) @@ -131,7 +131,7 @@ async fn fetch_balance( let balance_as_big_decimal = u256_to_big_decimal(balance_as_u256, decimals).map_err(|e| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error: e.into(), + error: e.into_inner().into(), })?; Ok(BalanceData { @@ -153,8 +153,6 @@ impl EventStreamer for EthBalanceEventStreamer { ready_tx: oneshot::Sender>, _: impl StreamHandlerInput, ) { - const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; - async fn start_polling(streamer_id: String, broadcaster: Broadcaster, coin: EthCoin, interval: f64) { async fn sleep_remaining_time(interval: f64, now: Instant) { // If the interval is x seconds, @@ -210,8 +208,7 @@ impl EventStreamer for EthBalanceEventStreamer { err.address, err.error ); - let e = serde_json::to_value(err.error).expect("Serialization shouldn't fail."); - // FIXME: We should add the address in the error message. + let e = serde_json::to_value(err).expect("Serialization shouldn't fail."); broadcaster.broadcast(Event::err(streamer_id.clone(), e)); }, }; @@ -225,7 +222,9 @@ impl EventStreamer for EthBalanceEventStreamer { } } - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); start_polling(self.streamer_id(), broadcaster, self.coin, self.interval).await } From bfb15daa7a76c52c679f79b93e2a48af28bfaaf3 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 20:11:00 +0200 Subject: [PATCH 068/128] let the manager receive reference through `send` and only clone it if the streamer exists (and make sure it injests inputs [data_in], but this really shouldn't happen as this would be a dev error --- mm2src/mm2_event_stream/src/manager.rs | 6 ++---- mm2src/mm2_main/src/lp_ordermatch.rs | 8 ++++---- mm2src/mm2_main/src/lp_ordermatch/order_events.rs | 2 +- mm2src/mm2_main/src/lp_swap/maker_swap.rs | 2 +- mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs | 2 +- mm2src/mm2_main/src/lp_swap/swap_events.rs | 2 +- mm2src/mm2_main/src/lp_swap/taker_swap.rs | 2 +- mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs | 2 +- 8 files changed, 12 insertions(+), 14 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 61c1078f74..b67f5698bb 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -143,16 +143,14 @@ impl StreamingManager { } /// Sends data to a streamer with `streamer_id`. - // FIXME: Consider receiving `data` as a `Clone`able &T and only clone it if streamer is found. - // This is to reduce the memory overhead when streamers are inactive. - pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { + pub fn send(&self, streamer_id: &str, data: &T) -> Result<(), StreamingManagerError> { let streamers = self.streamers.read().unwrap(); let streamer_info = streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; data_in - .unbounded_send(Box::new(data)) + .unbounded_send(Box::new(data.clone())) .map_err(|e| StreamingManagerError::SendError(e.to_string())) } diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index fdfa27131a..61f3b03c52 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -3569,7 +3569,7 @@ async fn process_maker_reserved(ctx: MmArc, from_pubkey: H256Json, reserved_msg: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::TakerMatch(taker_match.clone()), + &OrderStatusEvent::TakerMatch(taker_match.clone()), ) .ok(); @@ -3625,7 +3625,7 @@ async fn process_maker_connected(ctx: MmArc, from_pubkey: PublicKey, connected: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::TakerConnected(order_match.clone()), + &OrderStatusEvent::TakerConnected(order_match.clone()), ) .ok(); @@ -3739,7 +3739,7 @@ async fn process_taker_request(ctx: MmArc, from_pubkey: H256Json, taker_request: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::MakerMatch(maker_match.clone()), + &OrderStatusEvent::MakerMatch(maker_match.clone()), ) .ok(); @@ -3811,7 +3811,7 @@ async fn process_taker_connect(ctx: MmArc, sender_pubkey: PublicKey, connect_msg ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::MakerConnected(order_match.clone()), + &OrderStatusEvent::MakerConnected(order_match.clone()), ) .ok(); diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 1796beae26..037a173d97 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -11,7 +11,7 @@ impl OrderStatusStreamer { pub fn new() -> Self { Self } } -#[derive(Serialize)] +#[derive(Clone, Serialize)] #[serde(tag = "order_type", content = "order_data")] pub(super) enum OrderStatusEvent { MakerMatch(MakerMatch), diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index af76e7e8e5..4b1f3d61d9 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -2120,7 +2120,7 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { .expect("!save_my_maker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::MakerV1 { + .send(&SwapStatusStreamer.streamer_id(), &SwapStatusEvent::MakerV1 { uuid: running_swap.uuid, event: to_save, }) diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs index 0fc5a4ca72..078d9a1da0 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs @@ -721,7 +721,7 @@ impl Self { Self } } -#[derive(Serialize)] +#[derive(Clone, Serialize)] #[serde(tag = "swap_type", content = "swap_data")] pub enum SwapStatusEvent { MakerV1 { uuid: Uuid, event: MakerSavedEvent }, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap.rs b/mm2src/mm2_main/src/lp_swap/taker_swap.rs index b83e6fcc9d..3c6afacc97 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap.rs @@ -466,7 +466,7 @@ pub async fn run_taker_swap(swap: RunTakerSwapInput, ctx: MmArc) { .expect("!save_my_taker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::TakerV1 { + .send(&SwapStatusStreamer.streamer_id(), &SwapStatusEvent::TakerV1 { uuid: running_swap.uuid, event: to_save, }) diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs index 9611be533a..8003fa920a 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs @@ -841,7 +841,7 @@ impl Date: Mon, 29 Jul 2024 20:12:20 +0200 Subject: [PATCH 069/128] Revert "let the manager receive reference through `send`" This reverts commit bfb15daa7a76c52c679f79b93e2a48af28bfaaf3. Since we use custom enums for the streamers inputs that encapsulates other mm2 types. We ended up cloning the mm2 types to construct the enum and then clone the enums from the reference if the streamer exists. so we ended up double cloning instead of cloning just one time. --- mm2src/mm2_event_stream/src/manager.rs | 4 ++-- mm2src/mm2_main/src/lp_ordermatch.rs | 8 ++++---- mm2src/mm2_main/src/lp_ordermatch/order_events.rs | 2 +- mm2src/mm2_main/src/lp_swap/maker_swap.rs | 2 +- mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs | 2 +- mm2src/mm2_main/src/lp_swap/swap_events.rs | 2 +- mm2src/mm2_main/src/lp_swap/taker_swap.rs | 2 +- mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index b67f5698bb..60ebd5c2bf 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -143,14 +143,14 @@ impl StreamingManager { } /// Sends data to a streamer with `streamer_id`. - pub fn send(&self, streamer_id: &str, data: &T) -> Result<(), StreamingManagerError> { + pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { let streamers = self.streamers.read().unwrap(); let streamer_info = streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; data_in - .unbounded_send(Box::new(data.clone())) + .unbounded_send(Box::new(data)) .map_err(|e| StreamingManagerError::SendError(e.to_string())) } diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index 61f3b03c52..fdfa27131a 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -3569,7 +3569,7 @@ async fn process_maker_reserved(ctx: MmArc, from_pubkey: H256Json, reserved_msg: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - &OrderStatusEvent::TakerMatch(taker_match.clone()), + OrderStatusEvent::TakerMatch(taker_match.clone()), ) .ok(); @@ -3625,7 +3625,7 @@ async fn process_maker_connected(ctx: MmArc, from_pubkey: PublicKey, connected: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - &OrderStatusEvent::TakerConnected(order_match.clone()), + OrderStatusEvent::TakerConnected(order_match.clone()), ) .ok(); @@ -3739,7 +3739,7 @@ async fn process_taker_request(ctx: MmArc, from_pubkey: H256Json, taker_request: ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - &OrderStatusEvent::MakerMatch(maker_match.clone()), + OrderStatusEvent::MakerMatch(maker_match.clone()), ) .ok(); @@ -3811,7 +3811,7 @@ async fn process_taker_connect(ctx: MmArc, sender_pubkey: PublicKey, connect_msg ctx.event_stream_manager .send( &OrderStatusStreamer.streamer_id(), - &OrderStatusEvent::MakerConnected(order_match.clone()), + OrderStatusEvent::MakerConnected(order_match.clone()), ) .ok(); diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 037a173d97..1796beae26 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -11,7 +11,7 @@ impl OrderStatusStreamer { pub fn new() -> Self { Self } } -#[derive(Clone, Serialize)] +#[derive(Serialize)] #[serde(tag = "order_type", content = "order_data")] pub(super) enum OrderStatusEvent { MakerMatch(MakerMatch), diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index 4b1f3d61d9..af76e7e8e5 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -2120,7 +2120,7 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { .expect("!save_my_maker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), &SwapStatusEvent::MakerV1 { + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::MakerV1 { uuid: running_swap.uuid, event: to_save, }) diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs index 078d9a1da0..0fc5a4ca72 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs @@ -721,7 +721,7 @@ impl Self { Self } } -#[derive(Clone, Serialize)] +#[derive(Serialize)] #[serde(tag = "swap_type", content = "swap_data")] pub enum SwapStatusEvent { MakerV1 { uuid: Uuid, event: MakerSavedEvent }, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap.rs b/mm2src/mm2_main/src/lp_swap/taker_swap.rs index 3c6afacc97..b83e6fcc9d 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap.rs @@ -466,7 +466,7 @@ pub async fn run_taker_swap(swap: RunTakerSwapInput, ctx: MmArc) { .expect("!save_my_taker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), &SwapStatusEvent::TakerV1 { + .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::TakerV1 { uuid: running_swap.uuid, event: to_save, }) diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs index 8003fa920a..9611be533a 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs @@ -841,7 +841,7 @@ impl Date: Mon, 29 Jul 2024 20:38:22 +0200 Subject: [PATCH 070/128] using partking lot rwlock to not starve writers since we expect `StreamingManager::send` & `StreamingManager::broadcast` to be called alot thus holding read locks to `streamers` and/or `controller`, we might end up in a situation where a write upate can't take place because of a queue of readers --- mm2src/mm2_event_stream/src/manager.rs | 42 ++++++------------- .../src/rpc/streaming_activations/mod.rs | 4 +- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 60ebd5c2bf..8885c5c480 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,6 +1,6 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex}; use crate::streamer::Broadcaster; use crate::{controller::Controller, Event, EventStreamer}; @@ -9,6 +9,7 @@ use common::log; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; +use parking_lot::RwLock; use tokio::sync::mpsc::Receiver; /// The errors that could originate from the streaming manager. @@ -81,7 +82,6 @@ impl StreamingManager { // Pre-checks before spawning the streamer. Inside another scope to drop the locks early. { let mut clients = self.clients.lock().unwrap(); - let mut streamers = self.streamers.write().unwrap(); match clients.get(&client_id) { // We don't know that client. We don't have a connection to it. @@ -94,22 +94,13 @@ impl StreamingManager { } // If a streamer is already up and running, we won't spawn another one. - if let Some(streamer_info) = streamers.get_mut(&streamer_id) { + if let Some(streamer_info) = self.streamers.write().get_mut(&streamer_id) { // Register the client as a listener to the streamer. streamer_info.clients.insert(client_id); // Register the streamer as listened-to by the client. clients .get_mut(&client_id) .map(|listening_to| listening_to.insert(streamer_id.clone())); - // FIXME: The streamer running is probably running with different configuration. - // We might want to inform the client that the configuration they asked for wasn't - // applied and return the active configuration instead? - // - // Note that there is no way to (re)spawn a streamer with a different configuration - // unless then we want to have different streamer for each and every client! - // - // Another restricted solution is to not let clients set the streamer configuration - // and only allow it through MM2 configuration file. return Ok(streamer_id); } } @@ -129,7 +120,6 @@ impl StreamingManager { listening_to.insert(streamer_id.clone()); self.streamers .write() - .unwrap() .entry(streamer_id.clone()) .or_insert(streamer_info) .clients @@ -144,7 +134,7 @@ impl StreamingManager { /// Sends data to a streamer with `streamer_id`. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { - let streamers = self.streamers.read().unwrap(); + let streamers = self.streamers.read(); let streamer_info = streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; @@ -160,7 +150,7 @@ impl StreamingManager { if let Some(listening_to) = clients.get_mut(&client_id) { listening_to.remove(streamer_id); - let mut streamers = self.streamers.write().unwrap(); + let mut streamers = self.streamers.write(); streamers .get_mut(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)? @@ -183,19 +173,13 @@ impl StreamingManager { /// this method broadcasts an event to the listening *clients* directly, independently /// of any streamer (i.e. bypassing any streamer). pub fn broadcast(&self, event: Event) { - if let Some(client_ids) = self - .streamers - .read() - .unwrap() - .get(event.origin()) - .map(|info| &info.clients) - { - self.controller.read().unwrap().broadcast(event, Some(client_ids)) + if let Some(client_ids) = self.streamers.read().get(event.origin()).map(|info| &info.clients) { + self.controller.read().broadcast(event, Some(client_ids)) } } /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. - pub fn broadcast_all(&self, event: Event) { self.controller.read().unwrap().broadcast(event, None) } + pub fn broadcast_all(&self, event: Event) { self.controller.read().broadcast(event, None) } pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { let mut clients = self.clients.lock().unwrap(); @@ -205,7 +189,7 @@ impl StreamingManager { clients.insert(client_id, HashSet::new()); // Note that events queued in the channel are `Arc<` shared. // So a 1024 long buffer isn't actually heavy on memory. - Ok(self.controller.write().unwrap().create_channel(client_id, 1024)) + Ok(self.controller.write().create_channel(client_id, 1024)) } pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { @@ -213,7 +197,7 @@ impl StreamingManager { // Remove the client from our known-clients map. let listening_to = clients.remove(&client_id).ok_or(StreamingManagerError::UnknownClient)?; // Remove the client from all the streamers it was listening to. - let mut streamers = self.streamers.write().unwrap(); + let mut streamers = self.streamers.write(); for streamer_id in listening_to { if let Some(streamer_info) = streamers.get_mut(&streamer_id) { streamer_info.clients.remove(&client_id); @@ -228,7 +212,7 @@ impl StreamingManager { } } // Remove our channel with this client. - self.controller.write().unwrap().remove_channel(&client_id); + self.controller.write().remove_channel(&client_id); Ok(()) } @@ -239,10 +223,10 @@ impl StreamingManager { /// In this case, we need to remove the streamer and de-list it from all clients. fn remove_streamer_if_down(&self, streamer_id: &str) { let mut clients = self.clients.lock().unwrap(); - let mut streamers = self.streamers.write().unwrap(); + let mut streamers = self.streamers.write(); if let Some(streamer_info) = streamers.get(streamer_id) { - // Remove the streamer from all clients listening to it. if streamer_info.is_down() { + // Remove the streamer from all clients listening to it. for client_id in &streamer_info.clients { clients .get_mut(client_id) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index e8629a9a65..ca37dcad9b 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -20,7 +20,9 @@ pub use swaps::*; /// Note that we don't have a unified request. It is rather defined per event streaming activation. pub struct EnableStreamingResponse { pub streamer_id: String, - // FIXME: Consider returning the applied config here (might be different from the one the client requested). + // FIXME: If the the streamer was already running, it is probably running with different configuration. + // We might want to inform the client that the configuration they asked for wasn't applied and return + // the active configuration instead? // pub config: Json, } From 6c3ce0ec519e7e33996f3b66b955889d5aaa2b85 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 29 Jul 2024 21:32:11 +0200 Subject: [PATCH 071/128] remove eth gas estimator context rely only on the streaming version `gas_fee_estimator` was removed from coins file and is now set while enabling the streaming for the first time. One can choose to enable it in 'simiple' or 'provider' mode. provider mode uses 'gas_api' defined in mm2.json, if not found we always fail back to simple again. --- mm2src/coins/eth.rs | 34 +- .../coins/eth/fee_estimation/eip1559/mod.rs | 27 -- .../eth/fee_estimation/eth_fee_events.rs | 25 +- mm2src/coins/eth/for_tests.rs | 1 - mm2src/coins/eth/v2_activation.rs | 6 - .../coins/rpc_command/get_estimated_fees.rs | 332 ------------------ mm2src/coins/rpc_command/mod.rs | 1 - .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 6 - 8 files changed, 27 insertions(+), 405 deletions(-) delete mode 100644 mm2src/coins/rpc_command/get_estimated_fees.rs diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 83943b1c99..4ebdd7887d 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -157,8 +157,7 @@ use nonce::ParityNonce; pub mod fee_estimation; use fee_estimation::eip1559::{block_native::BlocknativeGasApiCaller, infura::InfuraGasApiCaller, - simple::FeePerGasSimpleEstimator, FeeEstimatorState, FeePerGasEstimated, GasApiConfig, - GasApiProvider}; + simple::FeePerGasSimpleEstimator, FeePerGasEstimated, GasApiConfig, GasApiProvider}; /// https://github.com/artemii235/etomic-swap/blob/master/contracts/EtomicSwap.sol /// Dev chain (195.201.137.5:8565) contract address: 0x83965C539899cC0F918552e5A26915de40ee8852 @@ -612,8 +611,6 @@ pub struct EthCoinImpl { /// consisting of the token address and token ID, separated by a comma. This field is essential for tracking the NFT assets /// information (chain & contract type, amount etc.), where ownership and amount, in ERC1155 case, might change over time. pub nfts_infos: Arc>>, - /// Context for eth fee per gas estimator loop. Created if coin supports fee per gas estimation - pub(crate) platform_fee_estimator_state: Arc, /// This spawner is used to spawn coin's related futures that should be aborted on coin deactivation /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, @@ -5059,25 +5056,20 @@ impl EthCoin { } /// Get gas base fee and suggest priority tip fees for the next block (see EIP-1559) - pub async fn get_eip1559_gas_fee(&self, gas_api_conf: Option) -> Web3RpcResult { + pub async fn get_eip1559_gas_fee(&self, use_simple: bool) -> Web3RpcResult { let coin = self.clone(); let history_estimator_fut = FeePerGasSimpleEstimator::estimate_fee_by_history(&coin); let ctx = MmArc::from_weak(&coin.ctx).ok_or_else(|| MmError::new(Web3RpcError::Internal("ctx is null".into())))?; - // If a `gas_api_conf` was provided, use that. - let gas_api_conf = if let Some(gas_api_config) = gas_api_conf { - gas_api_config - } else { - let gas_api_conf = ctx.conf["gas_api"].clone(); - if gas_api_conf.is_null() { - debug!("No eth gas api provider config, using only history estimator"); - return history_estimator_fut - .await - .map_err(|e| MmError::new(Web3RpcError::Internal(e.to_string()))); - } - json::from_value(gas_api_conf) - .map_err(|e| MmError::new(Web3RpcError::InvalidGasApiConfig(e.to_string())))? - }; + + let gas_api_conf = ctx.conf["gas_api"].clone(); + if gas_api_conf.is_null() || use_simple { + return history_estimator_fut + .await + .map_err(|e| MmError::new(Web3RpcError::Internal(e.to_string()))); + } + let gas_api_conf: GasApiConfig = json::from_value(gas_api_conf) + .map_err(|e| MmError::new(Web3RpcError::InvalidGasApiConfig(e.to_string())))?; let provider_estimator_fut = match gas_api_conf.provider { GasApiProvider::Infura => InfuraGasApiCaller::fetch_infura_fee_estimation(&gas_api_conf.url).boxed(), GasApiProvider::Blocknative => { @@ -5108,7 +5100,7 @@ impl EthCoin { Ok(PayForGasOption::Legacy(LegacyGasPrice { gas_price })) }, SwapTxFeePolicy::Low | SwapTxFeePolicy::Medium | SwapTxFeePolicy::High => { - let fee_per_gas = coin.get_eip1559_gas_fee(None).await?; + let fee_per_gas = coin.get_eip1559_gas_fee(false).await?; let pay_result = match swap_fee_policy { SwapTxFeePolicy::Low => PayForGasOption::Eip1559(Eip1559FeePerGas { max_fee_per_gas: fee_per_gas.low.max_fee_per_gas, @@ -6317,7 +6309,6 @@ pub async fn eth_coin_from_conf_and_request( // all spawned futures related to `ETH` coin will be aborted as well. let abortable_system = try_s!(ctx.abortable_system.create_subsystem()); - let platform_fee_estimator_state = FeeEstimatorState::init_fee_estimator(ctx, conf, &coin_type).await?; let max_eth_tx_type = get_max_eth_tx_type_conf(ctx, conf, &coin_type).await?; let coin = EthCoinImpl { @@ -6342,7 +6333,6 @@ pub async fn eth_coin_from_conf_and_request( address_nonce_locks, erc20_tokens_infos: Default::default(), nfts_infos: Default::default(), - platform_fee_estimator_state, abortable_system, }; diff --git a/mm2src/coins/eth/fee_estimation/eip1559/mod.rs b/mm2src/coins/eth/fee_estimation/eip1559/mod.rs index 3dac4c54bd..b4c3ffbfbc 100644 --- a/mm2src/coins/eth/fee_estimation/eip1559/mod.rs +++ b/mm2src/coins/eth/fee_estimation/eip1559/mod.rs @@ -3,11 +3,7 @@ pub mod block_native; pub mod infura; pub mod simple; -use crate::AsyncMutex; -use common::executor::AbortOnDropHandle; - use ethereum_types::U256; -use std::sync::Arc; use url::Url; const FEE_PER_GAS_LEVELS: usize = 3; @@ -91,26 +87,3 @@ pub struct FeePerGasEstimated { /// priority trend (up or down) pub priority_fee_trend: String, } - -/// Gas fee estimator loop context, runs a loop to estimate max fee and max priority fee per gas according to EIP-1559 for the next block -/// -/// This FeeEstimatorContext handles rpc requests which start and stop gas fee estimation loop and handles the loop itself. -/// FeeEstimatorContext keeps the latest estimated gas fees to return them on rpc request -pub(crate) struct FeeEstimatorContext { - /// Latest estimated gas fee values - pub(crate) estimated_fees: Arc>, - /// Handler for estimator loop graceful shutdown - pub(crate) abort_handler: AsyncMutex>, -} - -/// Gas fee estimator creation state -pub(crate) enum FeeEstimatorState { - /// Gas fee estimation not supported for this coin - CoinNotSupported, - /// Platform coin required to be enabled for gas fee estimation for this coin - PlatformCoinRequired, - /// Fee estimator created, use simple internal estimator - Simple(AsyncMutex), - /// Fee estimator created, use provider or simple internal estimator (if provider fails) - Provider(AsyncMutex), -} diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index 86001b833d..cda6e5e34d 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -1,4 +1,3 @@ -use super::eip1559::GasApiConfig; use crate::eth::EthCoin; use common::executor::Timer; use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; @@ -10,25 +9,30 @@ use serde::Deserialize; use serde_json::Value as Json; use std::convert::TryFrom; +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +/// Types of estimators available. +/// Simple - simple internal gas price estimator based on historical data. +/// Provider - gas price estimator using external provider (using gas api). +enum EstimatorType { + Simple, + Provider, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] struct EthFeeStreamingConfig { /// The time in seconds to wait before re-estimating the gas fees. pub estimate_every: f64, - /// Optional Gas API provider to be used to fetch gas prices. - /// If not provided, the "simple" (history based) provider will be used. - // FIXME: There is a similar field in MM2.json config, that one is the one used for swaps. - // Should we allow setting the one in the streamer like this? More flexible but will end - // up confusing users. What about setting `gas_api` with another end point and always use - // the one set. - pub gas_api_provider: Option, + /// The type of the estimator to use. + pub estimator_type: EstimatorType, } impl Default for EthFeeStreamingConfig { fn default() -> Self { Self { estimate_every: 15.0, - gas_api_provider: None, + estimator_type: EstimatorType::Simple, } } } @@ -62,11 +66,12 @@ impl EventStreamer for EthFeeEventStreamer { .send(Ok(())) .expect("Receiver is dropped, which should never happen."); + let use_simple = matches!(self.config.estimator_type, EstimatorType::Simple); loop { let now = Instant::now(); match self .coin - .get_eip1559_gas_fee(self.config.gas_api_provider.clone()) + .get_eip1559_gas_fee(use_simple) .await .map(serialized::FeePerGasEstimated::try_from) { diff --git a/mm2src/coins/eth/for_tests.rs b/mm2src/coins/eth/for_tests.rs index 9ea93188ea..4d19b5efc8 100644 --- a/mm2src/coins/eth/for_tests.rs +++ b/mm2src/coins/eth/for_tests.rs @@ -72,7 +72,6 @@ pub(crate) fn eth_coin_from_keypair( max_eth_tx_type: None, erc20_tokens_infos: Default::default(), nfts_infos: Arc::new(Default::default()), - platform_fee_estimator_state: Arc::new(FeeEstimatorState::CoinNotSupported), abortable_system: AbortableQueue::default(), })); (ctx, eth_coin) diff --git a/mm2src/coins/eth/v2_activation.rs b/mm2src/coins/eth/v2_activation.rs index 1a9b8b794c..5f3e25e5ca 100644 --- a/mm2src/coins/eth/v2_activation.rs +++ b/mm2src/coins/eth/v2_activation.rs @@ -392,7 +392,6 @@ impl EthCoin { platform: protocol.platform, token_addr: protocol.token_addr, }; - let platform_fee_estimator_state = FeeEstimatorState::init_fee_estimator(&ctx, &conf, &coin_type).await?; let max_eth_tx_type = get_max_eth_tx_type_conf(&ctx, &conf, &coin_type).await?; let token = EthCoinImpl { @@ -420,7 +419,6 @@ impl EthCoin { address_nonce_locks: self.address_nonce_locks.clone(), erc20_tokens_infos: Default::default(), nfts_infos: Default::default(), - platform_fee_estimator_state, abortable_system, }; @@ -455,7 +453,6 @@ impl EthCoin { let coin_type = EthCoinType::Nft { platform: self.ticker.clone(), }; - let platform_fee_estimator_state = FeeEstimatorState::init_fee_estimator(&ctx, &conf, &coin_type).await?; let max_eth_tx_type = get_max_eth_tx_type_conf(&ctx, &conf, &coin_type).await?; let global_nft = EthCoinImpl { @@ -480,7 +477,6 @@ impl EthCoin { address_nonce_locks: self.address_nonce_locks.clone(), erc20_tokens_infos: Default::default(), nfts_infos: Arc::new(AsyncMutex::new(nft_infos)), - platform_fee_estimator_state, abortable_system, }; Ok(EthCoin(Arc::new(global_nft))) @@ -591,7 +587,6 @@ pub async fn eth_coin_from_conf_and_request_v2( // all spawned futures related to `ETH` coin will be aborted as well. let abortable_system = ctx.abortable_system.create_subsystem()?; let coin_type = EthCoinType::Eth; - let platform_fee_estimator_state = FeeEstimatorState::init_fee_estimator(ctx, conf, &coin_type).await?; let max_eth_tx_type = get_max_eth_tx_type_conf(ctx, conf, &coin_type).await?; let coin = EthCoinImpl { @@ -616,7 +611,6 @@ pub async fn eth_coin_from_conf_and_request_v2( address_nonce_locks, erc20_tokens_infos: Default::default(), nfts_infos: Default::default(), - platform_fee_estimator_state, abortable_system, }; diff --git a/mm2src/coins/rpc_command/get_estimated_fees.rs b/mm2src/coins/rpc_command/get_estimated_fees.rs deleted file mode 100644 index cbe85b2df4..0000000000 --- a/mm2src/coins/rpc_command/get_estimated_fees.rs +++ /dev/null @@ -1,332 +0,0 @@ -//! RPCs to start/stop gas fee estimator and get estimated base and priority fee per gas -use crate::eth::fee_estimation::eip1559::{FeeEstimatorContext, FeeEstimatorState, FeePerGasEstimated}; -use crate::eth::{EthCoin, EthCoinType}; -use crate::{lp_coinfind_or_err, wei_to_gwei_decimal, AsyncMutex, CoinFindError, MmCoinEnum, NumConversError}; -use common::executor::{spawn_abortable, Timer}; -use common::log::debug; -use common::{HttpStatusCode, StatusCode}; -use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::prelude::*; -use mm2_number::BigDecimal; - -use serde::{Deserialize, Serialize}; -use serde_json::{self as json, Value as Json}; -use std::convert::{TryFrom, TryInto}; -use std::ops::Deref; -use std::sync::Arc; - -const FEE_ESTIMATOR_NAME: &str = "eth_gas_fee_estimator_loop"; - -/// Estimated fee per gas units -#[derive(Clone, Debug, Serialize)] -pub enum EstimationUnits { - Gwei, -} - -impl Default for EstimationUnits { - fn default() -> Self { Self::Gwei } -} - -/// Priority level estimated max fee per gas -#[derive(Clone, Debug, Default, Serialize)] -pub struct FeePerGasLevel { - /// estimated max priority tip fee per gas in gwei - pub max_priority_fee_per_gas: BigDecimal, - /// estimated max fee per gas in gwei - pub max_fee_per_gas: BigDecimal, - /// estimated transaction min wait time in mempool in ms for this priority level - pub min_wait_time: Option, - /// estimated transaction max wait time in mempool in ms for this priority level - pub max_wait_time: Option, -} - -/// External struct for estimated fee per gas for several priority levels, in gwei -/// low/medium/high levels are supported -#[derive(Default, Debug, Clone, Serialize)] -pub struct FeePerGasEstimatedExt { - /// base fee for the next block in gwei - pub base_fee: BigDecimal, - /// estimated low priority fee - pub low: FeePerGasLevel, - /// estimated medium priority fee - pub medium: FeePerGasLevel, - /// estimated high priority fee - pub high: FeePerGasLevel, - /// which estimator used - pub source: String, - /// base trend (up or down) - pub base_fee_trend: String, - /// priority trend (up or down) - pub priority_fee_trend: String, - /// fee units - pub units: EstimationUnits, -} - -impl TryFrom for FeePerGasEstimatedExt { - type Error = MmError; - - fn try_from(fees: FeePerGasEstimated) -> Result { - Ok(Self { - base_fee: wei_to_gwei_decimal!(fees.base_fee)?, - low: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_priority_fee_per_gas)?, - min_wait_time: fees.low.min_wait_time, - max_wait_time: fees.low.max_wait_time, - }, - medium: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_priority_fee_per_gas)?, - min_wait_time: fees.medium.min_wait_time, - max_wait_time: fees.medium.max_wait_time, - }, - high: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_priority_fee_per_gas)?, - min_wait_time: fees.high.min_wait_time, - max_wait_time: fees.high.max_wait_time, - }, - source: fees.source.to_string(), - base_fee_trend: fees.base_fee_trend, - priority_fee_trend: fees.priority_fee_trend, - units: EstimationUnits::Gwei, - }) - } -} - -#[derive(Debug, Display, Serialize, SerializeErrorType)] -#[serde(tag = "error_type", content = "error_data")] -pub enum FeeEstimatorError { - #[display(fmt = "No such coin {}", coin)] - NoSuchCoin { coin: String }, - #[display(fmt = "Gas fee estimation not supported for this coin")] - CoinNotSupported, - #[display(fmt = "Platform coin needs to be enabled for gas fee estimation")] - PlatformCoinRequired, - #[display(fmt = "Gas fee estimator is already started")] - AlreadyStarted, - #[display(fmt = "Transport error: {}", _0)] - Transport(String), - #[display(fmt = "Gas fee estimator is not running")] - NotRunning, - #[display(fmt = "Internal error: {}", _0)] - InternalError(String), -} - -impl HttpStatusCode for FeeEstimatorError { - fn status_code(&self) -> StatusCode { - match self { - FeeEstimatorError::NoSuchCoin { .. } - | FeeEstimatorError::CoinNotSupported - | FeeEstimatorError::PlatformCoinRequired - | FeeEstimatorError::AlreadyStarted - | FeeEstimatorError::NotRunning => StatusCode::BAD_REQUEST, - FeeEstimatorError::Transport(_) | FeeEstimatorError::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, - } - } -} - -impl From for FeeEstimatorError { - fn from(e: NumConversError) -> Self { FeeEstimatorError::InternalError(e.to_string()) } -} - -impl From for FeeEstimatorError { - fn from(e: String) -> Self { FeeEstimatorError::InternalError(e) } -} - -impl From for FeeEstimatorError { - fn from(e: CoinFindError) -> Self { - match e { - CoinFindError::NoSuchCoin { coin } => FeeEstimatorError::NoSuchCoin { coin }, - } - } -} - -/// Gas fee estimator configuration -#[derive(Deserialize)] -enum FeeEstimatorConf { - NotConfigured, - #[serde(rename = "simple")] - Simple, - #[serde(rename = "provider")] - Provider, -} - -impl Default for FeeEstimatorConf { - fn default() -> Self { Self::NotConfigured } -} - -impl FeeEstimatorState { - /// Creates gas FeeEstimatorContext if configured for this coin and chain id, otherwise returns None. - /// The created context object (or None) is wrapped into a FeeEstimatorState so a gas fee rpc caller may know the reason why it was not created - pub(crate) async fn init_fee_estimator( - ctx: &MmArc, - conf: &Json, - coin_type: &EthCoinType, - ) -> Result, String> { - let fee_estimator_json = conf["gas_fee_estimator"].clone(); - let fee_estimator_conf: FeeEstimatorConf = if !fee_estimator_json.is_null() { - try_s!(json::from_value(fee_estimator_json)) - } else { - Default::default() - }; - match (fee_estimator_conf, coin_type) { - (FeeEstimatorConf::Simple, EthCoinType::Eth) => { - let fee_estimator_state = FeeEstimatorState::Simple(FeeEstimatorContext::new()); - Ok(Arc::new(fee_estimator_state)) - }, - (FeeEstimatorConf::Provider, EthCoinType::Eth) => { - let fee_estimator_state = FeeEstimatorState::Provider(FeeEstimatorContext::new()); - Ok(Arc::new(fee_estimator_state)) - }, - (_, EthCoinType::Erc20 { platform, .. }) | (_, EthCoinType::Nft { platform, .. }) => { - let platform_coin = lp_coinfind_or_err(ctx, platform).await; - match platform_coin { - Ok(MmCoinEnum::EthCoin(eth_coin)) => Ok(eth_coin.platform_fee_estimator_state.clone()), - _ => Ok(Arc::new(FeeEstimatorState::PlatformCoinRequired)), - } - }, - (FeeEstimatorConf::NotConfigured, _) => Ok(Arc::new(FeeEstimatorState::CoinNotSupported)), - } - } -} - -impl FeeEstimatorContext { - fn new() -> AsyncMutex { - AsyncMutex::new(FeeEstimatorContext { - estimated_fees: Default::default(), - abort_handler: AsyncMutex::new(None), - }) - } - - /// Fee estimation update period in secs, basically equals to eth blocktime - const fn get_refresh_interval() -> f64 { 15.0 } - - fn get_estimator_ctx(coin: &EthCoin) -> Result<&AsyncMutex, MmError> { - match coin.platform_fee_estimator_state.deref() { - FeeEstimatorState::CoinNotSupported => MmError::err(FeeEstimatorError::CoinNotSupported), - FeeEstimatorState::PlatformCoinRequired => MmError::err(FeeEstimatorError::PlatformCoinRequired), - FeeEstimatorState::Simple(fee_estimator_ctx) | FeeEstimatorState::Provider(fee_estimator_ctx) => { - Ok(fee_estimator_ctx) - }, - } - } - - async fn start_if_not_running(coin: &EthCoin) -> Result<(), MmError> { - let estimator_ctx = Self::get_estimator_ctx(coin)?; - let estimator_ctx = estimator_ctx.lock().await; - let mut handler = estimator_ctx.abort_handler.lock().await; - if handler.is_some() { - return MmError::err(FeeEstimatorError::AlreadyStarted); - } - *handler = Some(spawn_abortable(Self::fee_estimator_loop(coin.clone()))); - Ok(()) - } - - async fn request_to_stop(coin: &EthCoin) -> Result<(), MmError> { - let estimator_ctx = Self::get_estimator_ctx(coin)?; - let estimator_ctx = estimator_ctx.lock().await; - let mut handle_guard = estimator_ctx.abort_handler.lock().await; - // Handler will be dropped here, stopping the spawned loop immediately - handle_guard - .take() - .map(|_| ()) - .or_mm_err(|| FeeEstimatorError::NotRunning) - } - - async fn get_estimated_fees(coin: &EthCoin) -> Result> { - let estimator_ctx = Self::get_estimator_ctx(coin)?; - let estimator_ctx = estimator_ctx.lock().await; - let estimated_fees = estimator_ctx.estimated_fees.lock().await; - Ok(estimated_fees.clone()) - } - - async fn check_if_estimator_supported(ctx: &MmArc, ticker: &str) -> Result> { - let eth_coin = match lp_coinfind_or_err(ctx, ticker).await? { - MmCoinEnum::EthCoin(eth) => eth, - _ => return MmError::err(FeeEstimatorError::CoinNotSupported), - }; - let _ = Self::get_estimator_ctx(ð_coin)?; - Ok(eth_coin) - } - - /// Loop polling gas fee estimator - /// - /// This loop periodically calls get_eip1559_gas_fee which fetches fee per gas estimations from a gas api provider or calculates them internally - /// The retrieved data are stored in the fee estimator context - /// To connect to the chain and gas api provider the web3 instances are used from an EthCoin coin passed in the start rpc param, - /// so this coin must be enabled first. - /// Once the loop started any other EthCoin in mainnet may request fee estimations. - /// It is up to GUI to start and stop the loop when it needs it (considering that the data in context may be used - /// for any coin with Eth or Erc20 type from the mainnet). - async fn fee_estimator_loop(coin: EthCoin) { - loop { - let started = common::now_float(); - if let Ok(estimator_ctx) = Self::get_estimator_ctx(&coin) { - let estimated_fees = coin.get_eip1559_gas_fee(None).await.unwrap_or_default(); - let estimator_ctx = estimator_ctx.lock().await; - *estimator_ctx.estimated_fees.lock().await = estimated_fees; - } - - let elapsed = common::now_float() - started; - debug!("{FEE_ESTIMATOR_NAME} call to provider processed in {} seconds", elapsed); - - let wait_secs = FeeEstimatorContext::get_refresh_interval() - elapsed; - let wait_secs = if wait_secs < 0.0 { 0.0 } else { wait_secs }; - Timer::sleep(wait_secs).await; - } - } -} - -/// Rpc request to start or stop gas fee estimator -#[derive(Deserialize)] -pub struct FeeEstimatorStartStopRequest { - coin: String, -} - -/// Rpc response to request to start or stop gas fee estimator -#[derive(Serialize)] -pub struct FeeEstimatorStartStopResponse { - result: String, -} - -pub type FeeEstimatorStartStopResult = Result>; - -/// Rpc request to get latest estimated fee per gas -#[derive(Deserialize)] -pub struct FeeEstimatorRequest { - /// coin ticker - coin: String, -} - -pub type FeeEstimatorResult = Result>; - -/// Start gas priority fee estimator loop -pub async fn start_eth_fee_estimator(ctx: MmArc, req: FeeEstimatorStartStopRequest) -> FeeEstimatorStartStopResult { - let coin = FeeEstimatorContext::check_if_estimator_supported(&ctx, &req.coin).await?; - FeeEstimatorContext::start_if_not_running(&coin).await?; - Ok(FeeEstimatorStartStopResponse { - result: "Success".to_string(), - }) -} - -/// Stop gas priority fee estimator loop -pub async fn stop_eth_fee_estimator(ctx: MmArc, req: FeeEstimatorStartStopRequest) -> FeeEstimatorStartStopResult { - let coin = FeeEstimatorContext::check_if_estimator_supported(&ctx, &req.coin).await?; - FeeEstimatorContext::request_to_stop(&coin).await?; - Ok(FeeEstimatorStartStopResponse { - result: "Success".to_string(), - }) -} - -/// Get latest estimated fee per gas for a eth coin -/// -/// Estimation loop for this coin must be stated. -/// Only main chain is supported -/// -/// Returns latest estimated fee per gas for the next block -pub async fn get_eth_estimated_fee_per_gas(ctx: MmArc, req: FeeEstimatorRequest) -> FeeEstimatorResult { - let coin = FeeEstimatorContext::check_if_estimator_supported(&ctx, &req.coin).await?; - let estimated_fees = FeeEstimatorContext::get_estimated_fees(&coin).await?; - estimated_fees.try_into().mm_err(Into::into) -} diff --git a/mm2src/coins/rpc_command/mod.rs b/mm2src/coins/rpc_command/mod.rs index 0bec5ef493..c401853b2d 100644 --- a/mm2src/coins/rpc_command/mod.rs +++ b/mm2src/coins/rpc_command/mod.rs @@ -1,7 +1,6 @@ pub mod account_balance; pub mod get_current_mtp; pub mod get_enabled_coins; -pub mod get_estimated_fees; pub mod get_new_address; pub mod hd_account_balance_rpc_error; pub mod init_account_balance; diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 45777ea308..d9145a4264 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -18,8 +18,6 @@ use coins::rpc_command::tendermint::{ibc_chains, ibc_transfer_channels}; use coins::rpc_command::{account_balance::account_balance, get_current_mtp::get_current_mtp_rpc, get_enabled_coins::get_enabled_coins, - get_estimated_fees::{get_eth_estimated_fee_per_gas, start_eth_fee_estimator, - stop_eth_fee_estimator}, get_new_address::{cancel_get_new_address, get_new_address, init_get_new_address, init_get_new_address_status, init_get_new_address_user_action}, init_account_balance::{cancel_account_balance, init_account_balance, @@ -219,10 +217,6 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, ibc_chains).await, "ibc_transfer_channels" => handle_mmrpc(ctx, request, ibc_transfer_channels).await, "withdraw_nft" => handle_mmrpc(ctx, request, withdraw_nft).await, - // FIXME: Remove these methods? - // "start_eth_fee_estimator" => handle_mmrpc(ctx, request, start_eth_fee_estimator).await, - // "stop_eth_fee_estimator" => handle_mmrpc(ctx, request, stop_eth_fee_estimator).await, - // "get_eth_estimated_fee_per_gas" => handle_mmrpc(ctx, request, get_eth_estimated_fee_per_gas).await, "get_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, get_swap_transaction_fee_policy).await, "set_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, set_swap_transaction_fee_policy).await, "send_asked_data" => handle_mmrpc(ctx, request, send_asked_data_rpc).await, From d31ebad260051bbd54f3703311dd1384774cc99f Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 30 Jul 2024 14:43:48 +0200 Subject: [PATCH 072/128] add unit tests for steaming manager --- mm2src/mm2_event_stream/src/event.rs | 3 + mm2src/mm2_event_stream/src/manager.rs | 195 +++++++++++++++++++++++- mm2src/mm2_event_stream/src/streamer.rs | 122 +++++++++++++++ 3 files changed, 318 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index f61818c383..93b1918d6a 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -1,5 +1,8 @@ use serde_json::Value as Json; +// Note `Event` shouldn't be `Clone`able, but rather Arc/Rc wrapped and then shared. +// This is only for testing. +#[cfg_attr(test, derive(Clone, Debug, PartialEq))] #[derive(Default)] /// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 8885c5c480..3b6ec67497 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,8 +2,7 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; -use crate::streamer::Broadcaster; -use crate::{controller::Controller, Event, EventStreamer}; +use crate::{controller::Controller, Broadcaster, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; @@ -238,3 +237,195 @@ impl StreamingManager { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::streamer::test_utils::{InitErrorStreamer, PeriodicStreamer, ReactiveStreamer}; + + use common::executor::{abortable_queue::AbortableQueue, AbortableSystem, Timer}; + use serde_json::json; + + #[tokio::test] + async fn test_add_remove_client() { + let manager = StreamingManager::default(); + let client_id1 = 1; + let client_id2 = 2; + let client_id3 = 3; + + assert!(matches!(manager.new_client(client_id1), Ok(..))); + // Adding the same client again should fail. + assert!(matches!( + manager.new_client(client_id1), + Err(StreamingManagerError::ClientExists) + )); + // Adding a different new client should be OK. + assert!(matches!(manager.new_client(client_id2), Ok(..))); + + assert!(matches!(manager.remove_client(client_id1), Ok(()))); + // Removing a removed client should fail. + assert!(matches!( + manager.remove_client(client_id1), + Err(StreamingManagerError::UnknownClient) + )); + // Same as removing a non-existent client. + assert!(matches!( + manager.remove_client(client_id3), + Err(StreamingManagerError::UnknownClient) + )); + } + + #[tokio::test] + async fn test_broadcast_all() { + // Create a manager and add register two clients with it. + let manager = StreamingManager::default(); + let mut client1 = manager.new_client(1).unwrap(); + let mut client2 = manager.new_client(2).unwrap(); + let event = Event::new("test".to_string(), json!("test")); + + // Broadcast the event to all clients. + manager.broadcast_all(event.clone()); + + // The clients should receive the events. + assert_eq!(*client1.try_recv().unwrap(), event); + assert_eq!(*client2.try_recv().unwrap(), event); + + // Remove the clients. + manager.remove_client(1).unwrap(); + manager.remove_client(2).unwrap(); + + // `recv` shouldn't work at this point since the client is removed. + assert!(client1.try_recv().is_err()); + assert!(client2.try_recv().is_err()); + } + + #[tokio::test] + async fn test_periodic_streamer() { + let manager = StreamingManager::default(); + let system = AbortableQueue::default(); + let (client_id1, client_id2) = (1, 2); + // Register a new client with the manager. + let mut client1 = manager.new_client(client_id1).unwrap(); + // Another client whom we won't have it subscribe to the streamer. + let mut client2 = manager.new_client(client_id2).unwrap(); + // Subscribe the new client to PeriodicStreamer. + let streamer_id = manager + .add(client_id1, PeriodicStreamer, system.weak_spawner()) + .await + .unwrap(); + + // We should be hooked now. try to receive some events from the streamer. + for _ in 0..3 { + // The streamer should send an event every 0.1s. Wait for 0.2s for safety. + Timer::sleep(0.15).await; + let event = client1.try_recv().unwrap(); + assert_eq!(event.origin(), streamer_id); + } + + // The other client shouldn't have received any events. + assert!(client2.try_recv().is_err()); + } + + #[tokio::test] + async fn test_reactive_streamer() { + let manager = StreamingManager::default(); + let system = AbortableQueue::default(); + let (client_id1, client_id2) = (1, 2); + // Register a new client with the manager. + let mut client1 = manager.new_client(client_id1).unwrap(); + // Another client whom we won't have it subscribe to the streamer. + let mut client2 = manager.new_client(client_id2).unwrap(); + // Subscribe the new client to ReactiveStreamer. + let streamer_id = manager + .add(client_id1, ReactiveStreamer, system.weak_spawner()) + .await + .unwrap(); + + // We should be hooked now. try to receive some events from the streamer. + for i in 1..=3 { + let msg = format!("send{}", i); + manager.send(&streamer_id, msg.clone()).unwrap(); + // Wait for a little bit to make sure the streamer received the data we sent. + Timer::sleep(0.1).await; + // The streamer should broadcast some event to the subscribed clients. + let event = client1.try_recv().unwrap(); + assert_eq!(event.origin(), streamer_id); + // It's an echo streamer, so the message should be the same. + assert_eq!(event.get().1, json!(msg)); + } + + // If we send the wrong datatype (void here instead of String), the streamer should ignore it. + manager.send(&streamer_id, ()).unwrap(); + Timer::sleep(0.1).await; + assert!(client1.try_recv().is_err()); + + // The other client shouldn't have received any events. + assert!(client2.try_recv().is_err()); + } + + #[tokio::test] + async fn test_erroring_streamer() { + let manager = StreamingManager::default(); + let system = AbortableQueue::default(); + let client_id = 1; + // Register a new client with the manager. + let _client = manager.new_client(client_id).unwrap(); + // Subscribe the new client to InitErrorStreamer. + let error = manager + .add(client_id, InitErrorStreamer, system.weak_spawner()) + .await + .unwrap_err(); + + assert!(matches!(error, StreamingManagerError::SpawnError(..))); + } + + #[tokio::test] + async fn test_remove_streamer_if_down() { + let manager = StreamingManager::default(); + let system = AbortableQueue::default(); + let client_id = 1; + // Register a new client with the manager. + let _client = manager.new_client(client_id).unwrap(); + // Subscribe the new client to PeriodicStreamer. + let streamer_id = manager + .add(client_id, PeriodicStreamer, system.weak_spawner()) + .await + .unwrap(); + + // The streamer is up and streaming to `client_id`. + assert!(manager + .streamers + .read() + .get(&streamer_id) + .unwrap() + .clients + .contains(&client_id)); + + // The client should be registered and listening to `streamer_id`. + assert!(manager + .clients + .lock() + .unwrap() + .get(&client_id) + .unwrap() + .contains(&streamer_id)); + + // Abort the system to kill the streamer. + system.abort_all().unwrap(); + // Wait a little bit since the abortion doesn't take effect immediately (the aborted task needs to yield first). + Timer::sleep(0.1).await; + + manager.remove_streamer_if_down(&streamer_id); + + // The streamer should be removed. + assert!(manager.streamers.read().get(&streamer_id).is_none()); + // And the client is no more listening to it. + assert!(!manager + .clients + .lock() + .unwrap() + .get(&client_id) + .unwrap() + .contains(&streamer_id)); + } +} diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 0a74b70313..d552749158 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -111,3 +111,125 @@ impl Broadcaster { pub fn broadcast(&self, event: Event) { self.0.broadcast(event); } } + +#[cfg(test)] +pub mod test_utils { + use super::*; + + use common::executor::Timer; + use serde_json::json; + + /// A test event streamer that broadcasts an event periodically. + /// Broadcasts `json!("hello")` every tenth of a second. + pub struct PeriodicStreamer; + + #[async_trait] + impl EventStreamer for PeriodicStreamer { + type DataInType = NoDataIn; + + fn streamer_id(&self) -> String { "periodic_streamer".to_string() } + + async fn handle( + self, + broadcaster: Broadcaster, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { + ready_tx.send(Ok(())).unwrap(); + loop { + broadcaster.broadcast(Event::new(self.streamer_id(), json!("hello"))); + Timer::sleep(0.1).await; + } + } + } + + /// A test event streamer that broadcasts an event whenever it receives a new message through `data_rx`. + pub struct ReactiveStreamer; + + #[async_trait] + impl EventStreamer for ReactiveStreamer { + type DataInType = String; + + fn streamer_id(&self) -> String { "reactive_streamer".to_string() } + + async fn handle( + self, + broadcaster: Broadcaster, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx.send(Ok(())).unwrap(); + while let Some(msg) = data_rx.next().await { + // Just echo back whatever we receive. + broadcaster.broadcast(Event::new(self.streamer_id(), json!(msg))); + } + } + } + + /// A test event streamer that fails upon initialization. + pub struct InitErrorStreamer; + + #[async_trait] + impl EventStreamer for InitErrorStreamer { + type DataInType = NoDataIn; + + fn streamer_id(&self) -> String { "init_error_streamer".to_string() } + + async fn handle( + self, + _: Broadcaster, + ready_tx: oneshot::Sender>, + _: impl StreamHandlerInput, + ) { + // Fail the initialization and stop. + ready_tx.send(Err("error".to_string())).unwrap(); + } + } +} + +#[cfg(test)] +mod tests { + use super::test_utils::{InitErrorStreamer, PeriodicStreamer, ReactiveStreamer}; + use super::*; + + use common::executor::abortable_queue::AbortableQueue; + + #[tokio::test] + async fn test_spawn_periodic_streamer() { + let system = AbortableQueue::default(); + let broadcaster = Broadcaster::new(StreamingManager::default()); + // Spawn the periodic streamer. + let (_, data_in) = PeriodicStreamer + .spawn(system.weak_spawner(), broadcaster) + .await + .unwrap(); + // Periodic streamer shouldn't be ingesting any input. + assert!(data_in.is_none()); + } + + #[tokio::test] + async fn test_spawn_reactive_streamer() { + let system = AbortableQueue::default(); + let broadcaster = Broadcaster::new(StreamingManager::default()); + // Spawn the reactive streamer. + let (_, data_in) = ReactiveStreamer + .spawn(system.weak_spawner(), broadcaster) + .await + .unwrap(); + // Reactive streamer should be ingesting some input. + assert!(data_in.is_some()); + } + + #[tokio::test] + async fn test_spawn_erroring_streamer() { + let system = AbortableQueue::default(); + let broadcaster = Broadcaster::new(StreamingManager::default()); + // Try to spawn the erroring streamer. + let err = InitErrorStreamer + .spawn(system.weak_spawner(), broadcaster) + .await + .unwrap_err(); + // The streamer should return an error. + assert_eq!(err, "error"); + } +} From 3b8876ded3c95f7a9e56a5f1dbf4d0dc15e8fe67 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 1 Aug 2024 10:12:16 +0200 Subject: [PATCH 073/128] add `send_fn` to streaming manager to minimize cloning so that we won't create message/notification we want to send to the streamer unless the streamer actually exists and accepts messages (if it doesn't accept messages, i.e. periodic streamer, this would definitely be a dev error) --- mm2src/mm2_event_stream/src/manager.rs | 20 +++++++++++++ mm2src/mm2_main/src/lp_ordermatch.rs | 29 ++++++++----------- .../src/lp_ordermatch/order_events.rs | 4 ++- mm2src/mm2_main/src/lp_swap/maker_swap.rs | 11 ++++--- mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs | 3 +- mm2src/mm2_main/src/lp_swap/swap_events.rs | 4 ++- mm2src/mm2_main/src/lp_swap/taker_swap.rs | 11 ++++--- mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs | 3 +- 8 files changed, 50 insertions(+), 35 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 3b6ec67497..9538637c91 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -143,6 +143,26 @@ impl StreamingManager { .map_err(|e| StreamingManagerError::SendError(e.to_string())) } + /// Same as `StreamingManager::send`, but computes that data to send to a streamer using a closure, + /// thus avoiding computations & cloning if the intended streamer isn't running (more like the + /// laziness of `*_or_else()` functions). + /// + /// `data_fn` will only be evaluated if the streamer is found and accepts an input. + pub fn send_fn( + &self, + streamer_id: &str, + data_fn: impl FnOnce() -> T, + ) -> Result<(), StreamingManagerError> { + let streamers = self.streamers.read(); + let streamer_info = streamers + .get(streamer_id) + .ok_or(StreamingManagerError::StreamerNotFound)?; + let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; + data_in + .unbounded_send(Box::new(data_fn())) + .map_err(|e| StreamingManagerError::SendError(e.to_string())) + } + /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { let mut clients = self.clients.lock().unwrap(); diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index fdfa27131a..cc37e15330 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -42,7 +42,6 @@ use http::Response; use keys::{AddressFormat, KeyPair}; use mm2_core::mm_ctx::{from_ctx, MmArc, MmWeak}; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_libp2p::{decode_signed, encode_and_sign, encode_message, pub_sub_topic, PublicKey, TopicHash, TopicPrefix, TOPIC_SEPARATOR}; use mm2_metrics::mm_gauge; @@ -3567,10 +3566,9 @@ async fn process_maker_reserved(ctx: MmArc, from_pubkey: H256Json, reserved_msg: }; ctx.event_stream_manager - .send( - &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::TakerMatch(taker_match.clone()), - ) + .send_fn(OrderStatusStreamer::derive_streamer_id(), || { + OrderStatusEvent::TakerMatch(taker_match.clone()) + }) .ok(); my_order @@ -3623,10 +3621,9 @@ async fn process_maker_connected(ctx: MmArc, from_pubkey: PublicKey, connected: } ctx.event_stream_manager - .send( - &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::TakerConnected(order_match.clone()), - ) + .send_fn(OrderStatusStreamer::derive_streamer_id(), || { + OrderStatusEvent::TakerConnected(order_match.clone()) + }) .ok(); // alice @@ -3737,10 +3734,9 @@ async fn process_taker_request(ctx: MmArc, from_pubkey: H256Json, taker_request: }; ctx.event_stream_manager - .send( - &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::MakerMatch(maker_match.clone()), - ) + .send_fn(OrderStatusStreamer::derive_streamer_id(), || { + OrderStatusEvent::MakerMatch(maker_match.clone()) + }) .ok(); order.matches.insert(maker_match.request.uuid, maker_match); @@ -3809,10 +3805,9 @@ async fn process_taker_connect(ctx: MmArc, sender_pubkey: PublicKey, connect_msg let order_match = order_match.clone(); ctx.event_stream_manager - .send( - &OrderStatusStreamer.streamer_id(), - OrderStatusEvent::MakerConnected(order_match.clone()), - ) + .send_fn(OrderStatusStreamer::derive_streamer_id(), || { + OrderStatusEvent::MakerConnected(order_match.clone()) + }) .ok(); my_order.started_swaps.push(order_match.request.uuid); diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 1796beae26..733d7bdba1 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -9,6 +9,8 @@ pub struct OrderStatusStreamer; impl OrderStatusStreamer { pub fn new() -> Self { Self } + + pub fn derive_streamer_id() -> &'static str { "ORDER_STATUS" } } #[derive(Serialize)] @@ -24,7 +26,7 @@ pub(super) enum OrderStatusEvent { impl EventStreamer for OrderStatusStreamer { type DataInType = (); - fn streamer_id(&self) -> String { "ORDER_STATUS".to_string() } + fn streamer_id(&self) -> String { Self::derive_streamer_id().to_string() } async fn handle( self, diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap.rs b/mm2src/mm2_main/src/lp_swap/maker_swap.rs index af76e7e8e5..18793141a3 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap.rs @@ -30,7 +30,6 @@ use futures::{compat::Future01CompatExt, select, FutureExt}; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; use mm2_rpc::data::legacy::OrderConfirmationsSettings; use parking_lot::Mutex as PaMutex; @@ -2115,16 +2114,16 @@ pub async fn run_maker_swap(swap: RunMakerSwapInput, ctx: MmArc) { .dispatch_async(ctx.clone(), LpEvents::MakerSwapStatusChanged(event_to_send)) .await; drop(dispatcher); - save_my_maker_swap_event(&ctx, &running_swap, to_save.clone()) - .await - .expect("!save_my_maker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::MakerV1 { + .send_fn(SwapStatusStreamer::derive_streamer_id(), || SwapStatusEvent::MakerV1 { uuid: running_swap.uuid, - event: to_save, + event: to_save.clone(), }) .ok(); + save_my_maker_swap_event(&ctx, &running_swap, to_save) + .await + .expect("!save_my_maker_swap_event"); if event.should_ban_taker() { ban_pubkey_on_failed_swap( &ctx, diff --git a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs index 0fc5a4ca72..65bf7acffc 100644 --- a/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/maker_swap_v2.rs @@ -21,7 +21,6 @@ use crypto::privkey::SerializableSecp256k1Keypair; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_libp2p::Secp256k1PubkeySerialize; use mm2_number::MmNumber; use mm2_state_machine::prelude::*; @@ -721,7 +720,7 @@ impl Self { Self } + + pub fn derive_streamer_id() -> &'static str { "SWAP_STATUS" } } #[derive(Serialize)] @@ -28,7 +30,7 @@ pub enum SwapStatusEvent { impl EventStreamer for SwapStatusStreamer { type DataInType = SwapStatusEvent; - fn streamer_id(&self) -> String { "SWAP_STATUS".to_string() } + fn streamer_id(&self) -> String { Self::derive_streamer_id().to_string() } async fn handle( self, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap.rs b/mm2src/mm2_main/src/lp_swap/taker_swap.rs index b83e6fcc9d..2e5a59acc7 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap.rs @@ -31,7 +31,6 @@ use http::Response; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_number::{BigDecimal, MmNumber}; use mm2_rpc::data::legacy::{MatchBy, OrderConfirmationsSettings, TakerAction}; use parking_lot::Mutex as PaMutex; @@ -461,16 +460,16 @@ pub async fn run_taker_swap(swap: RunTakerSwapInput, ctx: MmArc) { event: event.clone(), }; - save_my_taker_swap_event(&ctx, &running_swap, to_save.clone()) - .await - .expect("!save_my_taker_swap_event"); // Send a notification to the swap status streamer about a new event. ctx.event_stream_manager - .send(&SwapStatusStreamer.streamer_id(), SwapStatusEvent::TakerV1 { + .send_fn(SwapStatusStreamer::derive_streamer_id(), || SwapStatusEvent::TakerV1 { uuid: running_swap.uuid, - event: to_save, + event: to_save.clone(), }) .ok(); + save_my_taker_swap_event(&ctx, &running_swap, to_save) + .await + .expect("!save_my_taker_swap_event"); if event.should_ban_maker() { ban_pubkey_on_failed_swap( &ctx, diff --git a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs index 9611be533a..fbe1c6483b 100644 --- a/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs +++ b/mm2src/mm2_main/src/lp_swap/taker_swap_v2.rs @@ -22,7 +22,6 @@ use crypto::privkey::SerializableSecp256k1Keypair; use keys::KeyPair; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; -use mm2_event_stream::EventStreamer; use mm2_libp2p::Secp256k1PubkeySerialize; use mm2_number::MmNumber; use mm2_state_machine::prelude::*; @@ -841,7 +840,7 @@ impl Date: Thu, 1 Aug 2024 18:11:57 +0200 Subject: [PATCH 074/128] add tx (history) streaming this adds tx history streaming for tx history supporting coins (except zcoin). the streamer is fairly simple, just a forwarder that doesn't do any computation. The tx details are grapped from the tx history state machine and forwared to the streamer right before getting stored in the db. To be able to access the `event_stream_manager` from the tx history statemachine, `MmArc` was made accessible through the coins (`MmCoin`). Chose the coins here and not directly embedding the MmArc inside the history statemachine since we will probably use the event streamer more and more in different areas of the code, so having it distributed among coins would make adding new streaming functionalies easier. We could have also gone for cloning `event_stream_manager` to the coins instead of passing the `ctx`. Went for passing the ctx here since eth & tindermint already pass the ctx, so to avoid redundency and unify coins structure. --- mm2src/coins/eth.rs | 2 + mm2src/coins/lightning.rs | 2 + mm2src/coins/lp_coins.rs | 6 ++ mm2src/coins/qrc20.rs | 2 + mm2src/coins/sia.rs | 2 + mm2src/coins/solana.rs | 2 + mm2src/coins/solana/spl.rs | 2 + mm2src/coins/tendermint/tendermint_coin.rs | 2 + mm2src/coins/tendermint/tendermint_token.rs | 2 + .../tendermint/tendermint_tx_history_v2.rs | 9 ++ mm2src/coins/test_coin.rs | 2 + mm2src/coins/utxo.rs | 5 +- mm2src/coins/utxo/bch.rs | 2 + mm2src/coins/utxo/qtum.rs | 2 + mm2src/coins/utxo/slp.rs | 2 + mm2src/coins/utxo/tx_history_events.rs | 41 +++++++++ .../utxo/utxo_builder/utxo_coin_builder.rs | 2 + mm2src/coins/utxo/utxo_common_tests.rs | 1 + mm2src/coins/utxo/utxo_standard.rs | 2 + mm2src/coins/utxo/utxo_tx_history_v2.rs | 16 +++- mm2src/coins/z_coin.rs | 2 + .../storage/blockdb/blockdb_sql_storage.rs | 2 + .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 3 +- .../src/rpc/streaming_activations/mod.rs | 2 + .../rpc/streaming_activations/tx_history.rs | 87 +++++++++++++++++++ 25 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 mm2src/coins/utxo/tx_history_events.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 4ebdd7887d..be4684c68c 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5382,6 +5382,8 @@ impl EthTxFeeDetails { impl MmCoin for EthCoin { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.ctx) } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 1f8952c94a..dac5c7e9b7 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1275,6 +1275,8 @@ struct LightningProtocolInfo { impl MmCoin for LightningCoin { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { self.platform.coin.get_ctx() } + fn spawner(&self) -> WeakSpawner { self.platform.abortable_system.weak_spawner() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index fe5d562168..977d2e6c00 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -3220,6 +3220,12 @@ pub trait MmCoin: coin_conf["wallet_only"].as_bool().unwrap_or(false) } + /// Returns the MmArc context the coin is running on top of. + /// + /// This should really never fail (return `None`), but it's here since coins hold a weak + /// reference to the context (TODO: why?). + fn get_ctx(&self) -> Option; + /// Returns a spawner pinned to the coin. /// /// # Note diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 28f1a90748..f3a8442919 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -1313,6 +1313,8 @@ impl MarketCoinOps for Qrc20Coin { impl MmCoin for Qrc20Coin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo) } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins/sia.rs b/mm2src/coins/sia.rs index 1dfefb6369..216f9290d1 100644 --- a/mm2src/coins/sia.rs +++ b/mm2src/coins/sia.rs @@ -213,6 +213,8 @@ pub struct SiaCoinProtocolInfo; impl MmCoin for SiaCoin { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { unimplemented!() } + fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/solana.rs b/mm2src/coins/solana.rs index 4507d1a245..1fa762639e 100644 --- a/mm2src/coins/solana.rs +++ b/mm2src/coins/solana.rs @@ -997,6 +997,8 @@ impl WatcherOps for SolanaCoin { impl MmCoin for SolanaCoin { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { unimplemented!() } + fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins/solana/spl.rs b/mm2src/coins/solana/spl.rs index 64baf8438b..9aecd261a5 100644 --- a/mm2src/coins/solana/spl.rs +++ b/mm2src/coins/solana/spl.rs @@ -526,6 +526,8 @@ impl WatcherOps for SplToken { impl MmCoin for SplToken { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { self.platform_coin.get_ctx() } + fn spawner(&self) -> WeakSpawner { self.conf.abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins/tendermint/tendermint_coin.rs b/mm2src/coins/tendermint/tendermint_coin.rs index 2eccceffe5..0b9db434b7 100644 --- a/mm2src/coins/tendermint/tendermint_coin.rs +++ b/mm2src/coins/tendermint/tendermint_coin.rs @@ -2121,6 +2121,8 @@ pub async fn get_ibc_chain_list() -> IBCChainRegistriesResult { impl MmCoin for TendermintCoin { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.ctx) } + fn wallet_only(&self, ctx: &MmArc) -> bool { let coin_conf = crate::coin_conf(ctx, self.ticker()); let wallet_only_conf = coin_conf["wallet_only"].as_bool().unwrap_or(false); diff --git a/mm2src/coins/tendermint/tendermint_token.rs b/mm2src/coins/tendermint/tendermint_token.rs index cc7b99ddf4..983794be04 100644 --- a/mm2src/coins/tendermint/tendermint_token.rs +++ b/mm2src/coins/tendermint/tendermint_token.rs @@ -473,6 +473,8 @@ impl MarketCoinOps for TendermintToken { impl MmCoin for TendermintToken { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { self.platform_coin.get_ctx() } + fn wallet_only(&self, ctx: &MmArc) -> bool { let coin_conf = crate::coin_conf(ctx, self.ticker()); let wallet_only_conf = coin_conf["wallet_only"].as_bool().unwrap_or(false); diff --git a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs index 1c8adf8de3..ece9b6a89c 100644 --- a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs +++ b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs @@ -4,6 +4,7 @@ use crate::my_tx_history_v2::{CoinWithTxHistoryV2, MyTxHistoryErrorV2, MyTxHisto use crate::tendermint::htlc::CustomTendermintMsgType; use crate::tendermint::TendermintFeeDetails; use crate::tx_history_storage::{GetTxHistoryFilters, WalletId}; +use crate::utxo::tx_history_events::TxHistoryEventStreamer; use crate::utxo::utxo_common::big_decimal_from_sat_unsigned; use crate::{HistorySyncState, MarketCoinOps, MmCoin, TransactionData, TransactionDetails, TransactionType, TxFeeDetails}; @@ -754,6 +755,14 @@ where log::debug!("Tx '{}' successfully parsed.", tx.hash); } + coin.get_ctx() + .unwrap() + .event_stream_manager + .send_fn(&TxHistoryEventStreamer::derive_streamer_id(coin.ticker()), || { + tx_details.clone() + }) + .ok(); + try_or_return_stopped_as_err!( storage .add_transactions_to_history(&coin.history_wallet_id(), tx_details) diff --git a/mm2src/coins/test_coin.rs b/mm2src/coins/test_coin.rs index 496c21fde2..eac0c4ee9d 100644 --- a/mm2src/coins/test_coin.rs +++ b/mm2src/coins/test_coin.rs @@ -339,6 +339,8 @@ impl WatcherOps for TestCoin { impl MmCoin for TestCoin { fn is_asset_chain(&self) -> bool { unimplemented!() } + fn get_ctx(&self) -> Option { unimplemented!() } + fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 902840a8bb..ca19be7a13 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -33,6 +33,7 @@ pub mod slp; pub mod spv; pub mod swap_proto_v2_scripts; pub mod utxo_balance_events; +pub mod tx_history_events; pub mod utxo_block_header_storage; pub mod utxo_builder; pub mod utxo_common; @@ -67,7 +68,7 @@ pub use keys::{Address, AddressBuilder, AddressFormat as UtxoAddressFormat, Addr AddressScriptType, KeyPair, LegacyAddress, Private, Public, Secret}; #[cfg(not(target_arch = "wasm32"))] use lightning_invoice::Currency as LightningCurrency; -use mm2_core::mm_ctx::MmArc; +use mm2_core::mm_ctx::{MmArc, MmWeak}; use mm2_err_handle::prelude::*; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; @@ -614,6 +615,8 @@ pub struct UtxoCoinFields { /// The watcher/receiver of the block headers synchronization status, /// initialized only for non-native mode if spv is enabled for the coin. pub block_headers_status_watcher: Option>>, + /// The + pub ctx: MmWeak, /// This abortable system is used to spawn coin's related futures that should be aborted on coin deactivation /// and on [`MmArc::stop`]. pub abortable_system: AbortableQueue, diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index 1a4b6fa492..7655c3622e 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -1265,6 +1265,8 @@ impl MarketCoinOps for BchCoin { impl MmCoin for BchCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 44bf389459..f8798f16be 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -880,6 +880,8 @@ impl MarketCoinOps for QtumCoin { impl MmCoin for QtumCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index 0580d8dbdc..0e9e06b3dd 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -1597,6 +1597,8 @@ impl From for TxFeeDetails { impl MmCoin for SlpToken { fn is_asset_chain(&self) -> bool { false } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.conf.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/tx_history_events.rs b/mm2src/coins/utxo/tx_history_events.rs new file mode 100644 index 0000000000..902fb5baeb --- /dev/null +++ b/mm2src/coins/utxo/tx_history_events.rs @@ -0,0 +1,41 @@ +use crate::TransactionDetails; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; + +use async_trait::async_trait; +use futures::channel::oneshot; +use futures::StreamExt; + +pub struct TxHistoryEventStreamer { + coin: String, +} + +impl TxHistoryEventStreamer { + pub fn new(coin: String) -> Self { Self { coin } } + + pub fn derive_streamer_id(coin: &str) -> String { format!("TX_HISTORY:{coin}") } +} + +#[async_trait] +impl EventStreamer for TxHistoryEventStreamer { + type DataInType = Vec; + + fn streamer_id(&self) -> String { Self::derive_streamer_id(&self.coin) } + + async fn handle( + self, + broadcaster: Broadcaster, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + while let Some(new_txs) = data_rx.next().await { + for new_tx in new_txs { + let tx_details = serde_json::to_value(new_tx).expect("Serialization should't fail."); + broadcaster.broadcast(Event::new(self.streamer_id(), tx_details)); + } + } + } +} diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 62f12103e8..7a112ffb41 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -296,6 +296,7 @@ where check_utxo_maturity, block_headers_status_notifier, block_headers_status_watcher, + ctx: builder.ctx().clone().weak(), abortable_system, }; @@ -373,6 +374,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { check_utxo_maturity, block_headers_status_notifier, block_headers_status_watcher, + ctx: self.ctx().clone().weak(), abortable_system, }; Ok(coin) diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index f68f2ff475..9f3b1d7a1f 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -147,6 +147,7 @@ pub(super) fn utxo_coin_fields_for_test( check_utxo_maturity: false, block_headers_status_notifier: None, block_headers_status_watcher: None, + ctx: MmWeak::default(), abortable_system: AbortableQueue::default(), } } diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index 1cfd1aaf6e..0324c50c16 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -909,6 +909,8 @@ impl MarketCoinOps for UtxoStandardCoin { impl MmCoin for UtxoStandardCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/utxo_tx_history_v2.rs b/mm2src/coins/utxo/utxo_tx_history_v2.rs index 698a9bf0b6..8670ceaac8 100644 --- a/mm2src/coins/utxo/utxo_tx_history_v2.rs +++ b/mm2src/coins/utxo/utxo_tx_history_v2.rs @@ -4,10 +4,11 @@ use crate::my_tx_history_v2::{CoinWithTxHistoryV2, DisplayAddress, TxHistoryStor use crate::tx_history_storage::FilteringAddresses; use crate::utxo::bch::BchCoin; use crate::utxo::slp::ParseSlpScriptError; +use crate::utxo::tx_history_events::TxHistoryEventStreamer; use crate::utxo::{utxo_common, AddrFromStrError, GetBlockHeaderError}; use crate::{BalanceError, BalanceResult, BlockHeightAndTime, CoinWithDerivationMethod, HistorySyncState, - MarketCoinOps, NumConversError, ParseBigDecimalError, TransactionDetails, UnexpectedDerivationMethod, - UtxoRpcError, UtxoTx}; + MarketCoinOps, MmCoin, NumConversError, ParseBigDecimalError, TransactionDetails, + UnexpectedDerivationMethod, UtxoRpcError, UtxoTx}; use async_trait::async_trait; use common::executor::Timer; use common::log::{error, info}; @@ -104,7 +105,7 @@ pub struct UtxoTxDetailsParams<'a, Storage> { #[async_trait] pub trait UtxoTxHistoryOps: - CoinWithTxHistoryV2 + CoinWithDerivationMethod + MarketCoinOps + Send + Sync + 'static + CoinWithTxHistoryV2 + CoinWithDerivationMethod + MarketCoinOps + MmCoin + Send + Sync + 'static { /// Returns addresses for those we need to request Transaction history. async fn my_addresses(&self) -> MmResult, UtxoMyAddressesHistoryError>; @@ -620,6 +621,15 @@ where }, }; + ctx.coin + .get_ctx() + .unwrap() + .event_stream_manager + .send_fn(&TxHistoryEventStreamer::derive_streamer_id(ctx.coin.ticker()), || { + tx_details.clone() + }) + .ok(); + if let Err(e) = ctx.storage.add_transactions_to_history(&wallet_id, tx_details).await { return Self::change_state(Stopped::storage_error(e)); } diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index ad9bdd6134..3d5ce9a0ec 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1646,6 +1646,8 @@ impl WatcherOps for ZCoin { impl MmCoin for ZCoin { fn is_asset_chain(&self) -> bool { self.utxo_arc.conf.asset_chain } + fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } + fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, _req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs b/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs index 7523360807..456201eab4 100644 --- a/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs +++ b/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs @@ -230,6 +230,8 @@ impl BlockDbImpl { // If there are transactions present in the current scanned block, // we send a `Triggered` event to update the balance change. if tx_size > 0 { + // FIXME: WE want to convert this txs to tx details like other utxo coins + // : ctx.event_streamer_manager.send_fn("zcoin_history", || vector_of_txs).ok(); if let Some(mut sender) = z_balance_change_sender.clone() { sender.send(()).await.expect("No receiver is available/dropped"); }; diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index d9145a4264..5014ed85f8 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -197,6 +197,7 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, my_recent_swaps_rpc).await, "my_swap_status" => handle_mmrpc(ctx, request, my_swap_status_rpc).await, "my_tx_history" => handle_mmrpc(ctx, request, my_tx_history_v2_rpc).await, + "z_coin_tx_history" => handle_mmrpc(ctx, request, coins::my_tx_history_v2::z_coin_tx_history_rpc).await, "orderbook" => handle_mmrpc(ctx, request, orderbook_rpc_v2).await, "recreate_swap_data" => handle_mmrpc(ctx, request, recreate_swap_data).await, "refresh_nft_metadata" => handle_mmrpc(ctx, request, refresh_nft_metadata).await, @@ -220,7 +221,6 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, get_swap_transaction_fee_policy).await, "set_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, set_swap_transaction_fee_policy).await, "send_asked_data" => handle_mmrpc(ctx, request, send_asked_data_rpc).await, - "z_coin_tx_history" => handle_mmrpc(ctx, request, coins::my_tx_history_v2::z_coin_tx_history_rpc).await, #[cfg(not(target_arch = "wasm32"))] native_only_methods => match native_only_methods { #[cfg(all(feature = "enable-solana", not(target_os = "ios"), not(target_os = "android")))] @@ -331,6 +331,7 @@ async fn rpc_streaming_dispatcher( "fee_estimator::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_fee_estimation).await, "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, + "tx_history::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_tx_history).await, "disable" => handle_mmrpc(ctx, request, streaming_activations::disable_streamer).await, _ => MmError::err(DispatcherError::NoSuchMethod), } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index ca37dcad9b..c06311abaa 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -5,6 +5,7 @@ mod heartbeat; mod network; mod orders; mod swaps; +mod tx_history; // Re-exports pub use balance::*; @@ -14,6 +15,7 @@ pub use heartbeat::*; pub use network::*; pub use orders::*; pub use swaps::*; +pub use tx_history::*; #[derive(Serialize)] /// The success/ok response for any event streaming activation request. diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs new file mode 100644 index 0000000000..a8657a8f86 --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs @@ -0,0 +1,87 @@ +//! RPC activation and deactivation for Tx history event streamers. +use super::EnableStreamingResponse; + +use coins::utxo::tx_history_events::TxHistoryEventStreamer; +use coins::{lp_coinfind, MmCoin, MmCoinEnum}; +use common::HttpStatusCode; +use http::StatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + + +#[derive(Deserialize)] +pub struct EnableTxHistoryStreamingRequest { + pub client_id: u64, + pub coin: String, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum TxHistoryStreamingRequestError { + EnableError(String), + CoinNotFound, + CoinNotSupported, + Internal(String), +} + +impl HttpStatusCode for TxHistoryStreamingRequestError { + fn status_code(&self) -> StatusCode { + match self { + TxHistoryStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + TxHistoryStreamingRequestError::CoinNotFound => StatusCode::NOT_FOUND, + TxHistoryStreamingRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, + TxHistoryStreamingRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +pub async fn enable_tx_history( + ctx: MmArc, + req: EnableTxHistoryStreamingRequest, +) -> MmResult { + let coin = lp_coinfind(&ctx, &req.coin) + .await + .map_err(TxHistoryStreamingRequestError::Internal)? + .ok_or(TxHistoryStreamingRequestError::CoinNotFound)?; + + let enable_result = match coin { + MmCoinEnum::UtxoCoin(coin) => { + let streamer = TxHistoryEventStreamer::new(req.coin); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::Bch(coin) => { + let streamer = TxHistoryEventStreamer::new(req.coin); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::QtumCoin(coin) => { + let streamer = TxHistoryEventStreamer::new(req.coin); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + MmCoinEnum::Tendermint(coin) => { + // The tx history streamer is very primitive reactive streamer that only emits new txs. + // it's logic is exactly the same for utxo coins and tendermint coins as well. + let streamer = TxHistoryEventStreamer::new(req.coin); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, + // MmCoinEnum::ZCoin(coin) => { + // let streamer = ZCoinTxHistoryEventStreamer::new(req.coin); + // ctx.event_stream_manager + // .add(req.client_id, streamer, coin.spawner()) + // .await + // }, + // FIXME: What about tokens?! + _ => Err(TxHistoryStreamingRequestError::CoinNotSupported)?, + }; + + enable_result + .map(EnableStreamingResponse::new) + .map_to_mm(|e| TxHistoryStreamingRequestError::EnableError(format!("{e:?}"))) +} From 5cf1dd71ba317bee51d61f97ad0b55ba1f960072 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 1 Aug 2024 20:37:10 +0200 Subject: [PATCH 075/128] why use mut & remove? --- mm2src/coins/z_coin.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 3d5ce9a0ec..9e3f059313 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -519,7 +519,7 @@ impl ZCoin { fn tx_details_from_db_item( &self, tx_item: ZCoinTxHistoryItem, - transactions: &mut HashMap, + transactions: &HashMap, prev_transactions: &HashMap, current_block: u64, ) -> Result> { @@ -532,7 +532,7 @@ impl ZCoin { let mut transparent_input_amount = Amount::zero(); let hash = H256Json::from(tx_item.tx_hash.as_slice()); - let z_tx = transactions.remove(&hash).or_mm_err(|| NoInfoAboutTx(hash))?; + let z_tx = transactions.get(&hash).or_mm_err(|| NoInfoAboutTx(hash))?; for input in z_tx.vin.iter() { let mut hash = H256Json::from(*input.prevout.hash()); hash.0.reverse(); @@ -623,7 +623,7 @@ impl ZCoin { .iter() .map(|item| H256Json::from(item.tx_hash.as_slice())) .collect(); - let mut transactions = self.z_transactions_from_cache_or_rpc(hashes_for_verbose).await?; + let transactions = self.z_transactions_from_cache_or_rpc(hashes_for_verbose).await?; let prev_tx_hashes: HashSet<_> = transactions .iter() @@ -641,7 +641,7 @@ impl ZCoin { .transactions .into_iter() .map(|sql_item| { - self.tx_details_from_db_item(sql_item, &mut transactions, &prev_transactions, current_block) + self.tx_details_from_db_item(sql_item, &transactions, &prev_transactions, current_block) }) .collect::>()?; From fe0205084e25f222873db70d079a9ee0c6cb8822 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 1 Aug 2024 18:21:49 +0200 Subject: [PATCH 076/128] fail utxo balance streamer initilization if running on top of native rpc and not electrum. This so the error could be reported back to the client and not silently ignored. P.S. we could support native rpc balance events using a periodic polling technique like the one we have in eth for example --- mm2src/coins/utxo/utxo_balance_events.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index dcd18563b5..bb3b4a21ca 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -49,7 +49,7 @@ impl EventStreamer for UtxoBalanceEventStreamer { self, broadcaster: Broadcaster, ready_tx: oneshot::Sender>, - mut data_rx: impl StreamHandlerInput, + mut data_rx: impl StreamHandlerInput, ) { const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; let streamer_id = self.streamer_id(); @@ -93,10 +93,9 @@ impl EventStreamer for UtxoBalanceEventStreamer { } if coin.as_ref().rpc_client.is_native() { - log::warn!("Native RPC client is not supported for {streamer_id} event. Skipping event initialization."); - // We won't consider this an error but just an unsupported scenario and continue running. - ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); - panic!("Native RPC client is not supported for UtxoBalanceEventStreamer."); + let msg = "Native RPC client is not supported for UtxoBalanceEventStreamer."; + ready_tx.send(Err(msg.to_string())).expect(RECEIVER_DROPPED_MSG); + panic!("{}", msg); } ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); From 179b76a07c2d9532ec734cac91b1e775f623e718 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 2 Aug 2024 12:04:30 +0200 Subject: [PATCH 077/128] partial impl of zcoin tx history this is still missing the translation from txhash to json tx details (this need to be fetched from the db). this commit also reactivates the z balance streaming by using the streaming manager to send the balance change notification to the z balance event streamer (instead of sending the notification on the channel which no body was listening to after balance streaming refactor). --- mm2src/coins/z_coin.rs | 22 +-- mm2src/coins/z_coin/storage.rs | 10 +- .../storage/blockdb/blockdb_idb_storage.rs | 25 ++-- .../storage/blockdb/blockdb_sql_storage.rs | 24 ++-- mm2src/coins/z_coin/storage/blockdb/mod.rs | 1 - mm2src/coins/z_coin/tx_history_events.rs | 131 ++++++++++++++++++ mm2src/coins/z_coin/z_balance_streaming.rs | 4 +- mm2src/coins/z_coin/z_coin_errors.rs | 1 + mm2src/coins/z_coin/z_rpc.rs | 13 +- mm2src/coins/z_coin/z_tx_history.rs | 16 +++ .../rpc/streaming_activations/tx_history.rs | 14 +- 11 files changed, 199 insertions(+), 62 deletions(-) create mode 100644 mm2src/coins/z_coin/tx_history_events.rs diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 9e3f059313..a9fd085846 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,4 +1,5 @@ pub mod storage; +pub mod tx_history_events; pub mod z_balance_streaming; mod z_coin_errors; #[cfg(all(test, feature = "zhtlc-native-tests"))] @@ -24,7 +25,6 @@ use crate::utxo::{sat_from_big_decimal, utxo_common, ActualTxFee, AdditionalTxDa UtxoCommonOps, UtxoRpcMode, UtxoTxBroadcastOps, UtxoTxGenerationOps, VerboseTransactionFrom}; use crate::utxo::{UnsupportedAddr, UtxoFeeDetails}; use crate::z_coin::storage::{BlockDbImpl, WalletDbShared}; -use crate::z_coin::z_balance_streaming::ZBalanceEventHandler; use crate::z_coin::z_tx_history::{fetch_tx_history_from_db, ZCoinTxHistoryItem}; use crate::{BalanceError, BalanceFut, CheckIfMyPaymentSentArgs, CoinBalance, ConfirmPaymentInput, DexFee, @@ -208,7 +208,6 @@ pub struct ZCoinFields { light_wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, sync_state_connector: AsyncMutex, - z_balance_event_handler: Option, } impl Transaction for ZTransaction { @@ -640,9 +639,7 @@ impl ZCoin { let transactions = req_result .transactions .into_iter() - .map(|sql_item| { - self.tx_details_from_db_item(sql_item, &transactions, &prev_transactions, current_block) - }) + .map(|sql_item| self.tx_details_from_db_item(sql_item, &transactions, &prev_transactions, current_block)) .collect::>()?; Ok(MyTxHistoryResponseV2 { @@ -880,22 +877,11 @@ impl<'a> UtxoCoinBuilder for ZCoinBuilder<'a> { ); let blocks_db = self.init_blocks_db().await?; - let (z_balance_event_sender, z_balance_event_handler) = { - let (sender, receiver) = futures::channel::mpsc::unbounded(); - (Some(sender), Some(Arc::new(AsyncMutex::new(receiver)))) - }; let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { #[cfg(not(target_arch = "wasm32"))] ZcoinRpcMode::Native => { - init_native_client( - &self, - self.native_client()?, - blocks_db, - &z_spending_key, - z_balance_event_sender, - ) - .await? + init_native_client(&self, self.native_client()?, blocks_db, &z_spending_key).await? }, ZcoinRpcMode::Light { light_wallet_d_servers, @@ -910,7 +896,6 @@ impl<'a> UtxoCoinBuilder for ZCoinBuilder<'a> { sync_params, skip_sync_params.unwrap_or_default(), &z_spending_key, - z_balance_event_sender, ) .await? }, @@ -926,7 +911,6 @@ impl<'a> UtxoCoinBuilder for ZCoinBuilder<'a> { light_wallet_db, consensus_params: self.protocol_info.consensus_params, sync_state_connector, - z_balance_event_handler, }); Ok(ZCoin { utxo_arc, z_fields }) diff --git a/mm2src/coins/z_coin/storage.rs b/mm2src/coins/z_coin/storage.rs index 08e478f27a..364689b27a 100644 --- a/mm2src/coins/z_coin/storage.rs +++ b/mm2src/coins/z_coin/storage.rs @@ -1,4 +1,5 @@ use crate::z_coin::{ValidateBlocksError, ZcoinConsensusParams, ZcoinStorageError}; +use mm2_event_stream::StreamingManager; pub mod blockdb; pub use blockdb::*; @@ -10,7 +11,6 @@ pub(crate) use z_params::ZcashParamsWasmImpl; pub use walletdb::*; -use crate::z_coin::z_balance_streaming::ZBalanceEventSender; use mm2_err_handle::mm_error::MmResult; #[cfg(target_arch = "wasm32")] use walletdb::wasm::storage::DataConnStmtCacheWasm; @@ -60,7 +60,7 @@ pub struct CompactBlockRow { #[derive(Clone)] pub enum BlockProcessingMode { Validate, - Scan(DataConnStmtCacheWrapper, Option), + Scan(DataConnStmtCacheWrapper, StreamingManager), } /// Checks that the scanned blocks in the data database, when combined with the recent @@ -119,7 +119,7 @@ pub async fn scan_cached_block( params: &ZcoinConsensusParams, block: &CompactBlock, last_height: &mut BlockHeight, -) -> Result { +) -> Result>, ValidateBlocksError> { let mut data_guard = data.inner().clone(); // Fetch the ExtendedFullViewingKeys we are tracking let extfvks = data_guard.get_extended_full_viewing_keys().await?; @@ -203,9 +203,7 @@ pub async fn scan_cached_block( ); witnesses.extend(new_witnesses); - *last_height = current_height; - // If there are any transactions in the block, return the transaction count - Ok(txs.len()) + Ok(txs) } diff --git a/mm2src/coins/z_coin/storage/blockdb/blockdb_idb_storage.rs b/mm2src/coins/z_coin/storage/blockdb/blockdb_idb_storage.rs index cccf8cc0a9..826ed52bdd 100644 --- a/mm2src/coins/z_coin/storage/blockdb/blockdb_idb_storage.rs +++ b/mm2src/coins/z_coin/storage/blockdb/blockdb_idb_storage.rs @@ -1,9 +1,10 @@ use crate::z_coin::storage::{scan_cached_block, validate_chain, BlockDbImpl, BlockProcessingMode, CompactBlockRow, ZcoinConsensusParams, ZcoinStorageRes}; +use crate::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; +use crate::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; use crate::z_coin::z_coin_errors::ZcoinStorageError; use async_trait::async_trait; -use futures_util::SinkExt; use mm2_core::mm_ctx::MmArc; use mm2_db::indexed_db::{BeBigUint, ConstructibleDb, DbIdentifier, DbInstance, DbLocked, DbUpgrader, IndexedDb, IndexedDbBuilder, InitDbResult, MultiIndex, OnUpgradeResult, TableSignature}; @@ -221,6 +222,7 @@ impl BlockDbImpl { validate_from: Option<(BlockHeight, BlockHash)>, limit: Option, ) -> ZcoinStorageRes<()> { + let ticker = self.ticker.to_owned(); let mut from_height = match &mode { BlockProcessingMode::Validate => validate_from .map(|(height, _)| height) @@ -241,7 +243,7 @@ impl BlockDbImpl { if block.height() != cbr.height { return MmError::err(ZcoinStorageError::CorruptedData(format!( - "Block height {} did not match row's height field value {}", + "{ticker}, Block height {} did not match row's height field value {}", block.height(), cbr.height ))); @@ -251,14 +253,17 @@ impl BlockDbImpl { BlockProcessingMode::Validate => { validate_chain(block, &mut prev_height, &mut prev_hash).await?; }, - BlockProcessingMode::Scan(data, z_balance_change_sender) => { - let tx_size = scan_cached_block(data, ¶ms, &block, &mut from_height).await?; - // If there is/are transactions present in the current scanned block(s), - // we trigger a `Triggered` event to update the balance change. - if tx_size > 0 { - if let Some(mut sender) = z_balance_change_sender.clone() { - sender.send(()).await.expect("No receiver is available/dropped"); - }; + BlockProcessingMode::Scan(data, streaming_manager) => { + let txs = scan_cached_block(data, ¶ms, &block, &mut from_height).await?; + if !txs.is_empty() { + // Stream out the new transactions. + streaming_manager + .send(&ZCoinTxHistoryEventStreamer::derive_streamer_id(&ticker), txs) + .ok(); + // And also stream balance changes. + streaming_manager + .send(&ZCoinBalanceEventStreamer::derive_streamer_id(&ticker), ()) + .ok(); }; }, } diff --git a/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs b/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs index 456201eab4..e8ccdd2554 100644 --- a/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs +++ b/mm2src/coins/z_coin/storage/blockdb/blockdb_sql_storage.rs @@ -1,12 +1,13 @@ use crate::z_coin::storage::{scan_cached_block, validate_chain, BlockDbImpl, BlockProcessingMode, CompactBlockRow, ZcoinStorageRes}; +use crate::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; +use crate::z_coin::z_balance_streaming::ZCoinBalanceEventStreamer; use crate::z_coin::z_coin_errors::ZcoinStorageError; use crate::z_coin::ZcoinConsensusParams; use common::async_blocking; use db_common::sqlite::rusqlite::{params, Connection}; use db_common::sqlite::{query_single_row, run_optimization_pragmas, rusqlite}; -use futures_util::SinkExt; use itertools::Itertools; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; @@ -225,16 +226,17 @@ impl BlockDbImpl { BlockProcessingMode::Validate => { validate_chain(block, &mut prev_height, &mut prev_hash).await?; }, - BlockProcessingMode::Scan(data, z_balance_change_sender) => { - let tx_size = scan_cached_block(data, ¶ms, &block, &mut from_height).await?; - // If there are transactions present in the current scanned block, - // we send a `Triggered` event to update the balance change. - if tx_size > 0 { - // FIXME: WE want to convert this txs to tx details like other utxo coins - // : ctx.event_streamer_manager.send_fn("zcoin_history", || vector_of_txs).ok(); - if let Some(mut sender) = z_balance_change_sender.clone() { - sender.send(()).await.expect("No receiver is available/dropped"); - }; + BlockProcessingMode::Scan(data, streaming_manager) => { + let txs = scan_cached_block(data, ¶ms, &block, &mut from_height).await?; + if !txs.is_empty() { + // Stream out the new transactions. + streaming_manager + .send(&ZCoinTxHistoryEventStreamer::derive_streamer_id(&ticker), txs) + .ok(); + // And also stream balance changes. + streaming_manager + .send(&ZCoinBalanceEventStreamer::derive_streamer_id(&ticker), ()) + .ok(); }; }, } diff --git a/mm2src/coins/z_coin/storage/blockdb/mod.rs b/mm2src/coins/z_coin/storage/blockdb/mod.rs index 7e2ef49fe7..1b3676dbbd 100644 --- a/mm2src/coins/z_coin/storage/blockdb/mod.rs +++ b/mm2src/coins/z_coin/storage/blockdb/mod.rs @@ -18,7 +18,6 @@ pub struct BlockDbImpl { pub db: Arc>, #[cfg(target_arch = "wasm32")] pub db: SharedDb, - #[allow(unused)] ticker: String, } diff --git a/mm2src/coins/z_coin/tx_history_events.rs b/mm2src/coins/z_coin/tx_history_events.rs new file mode 100644 index 0000000000..22763b0a4d --- /dev/null +++ b/mm2src/coins/z_coin/tx_history_events.rs @@ -0,0 +1,131 @@ +use super::z_tx_history::fetch_txs_from_db; +use super::{NoInfoAboutTx, ZCoin, ZcoinTxDetails}; +use crate::utxo::rpc_clients::UtxoRpcError; +use crate::MarketCoinOps; +use common::log; +use mm2_err_handle::prelude::MmError; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; +use rpc::v1::types::H256 as H256Json; + +use async_trait::async_trait; +use futures::channel::oneshot; +use futures::compat::Future01CompatExt; +use futures::StreamExt; +use zcash_client_backend::wallet::WalletTx; +use zcash_primitives::sapling::Nullifier; + +pub struct ZCoinTxHistoryEventStreamer { + coin: ZCoin, +} + +impl ZCoinTxHistoryEventStreamer { + pub fn new(coin: ZCoin) -> Self { Self { coin } } + + pub fn derive_streamer_id(coin: &str) -> String { format!("TX_HISTORY:{coin}") } +} + +#[async_trait] +impl EventStreamer for ZCoinTxHistoryEventStreamer { + type DataInType = Vec>; + + fn streamer_id(&self) -> String { Self::derive_streamer_id(self.coin.ticker()) } + + async fn handle( + self, + broadcaster: Broadcaster, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + while let Some(new_txs) = data_rx.next().await { + let new_txs_details = match get_tx_details(&self.coin, new_txs).await { + Ok(tx_details) => tx_details, + Err(e) => { + broadcaster.broadcast(Event::err(self.streamer_id(), json!({ "error": e.to_string() }))); + log::error!("Failed to get tx details in streamer {}: {e:?}", self.streamer_id()); + continue; + }, + }; + for tx_details in new_txs_details { + let tx_details = serde_json::to_value(tx_details).expect("Serialization should't fail."); + broadcaster.broadcast(Event::new(self.streamer_id(), tx_details)); + } + } + } +} + +#[derive(Debug)] +enum GetTxDetailsError { + UtxoRpcError(UtxoRpcError), + #[cfg(not(target_arch = "wasm32"))] + SqliteDbError(db_common::sqlite::rusqlite::Error), + #[cfg(target_arch = "wasm32")] + IndexedDbError(String), + Internal(NoInfoAboutTx), +} + +impl From> for GetTxDetailsError { + fn from(e: MmError) -> Self { GetTxDetailsError::UtxoRpcError(e.into_inner()) } +} + +#[cfg(not(target_arch = "wasm32"))] +impl From for GetTxDetailsError { + fn from(e: db_common::sqlite::rusqlite::Error) -> Self { GetTxDetailsError::SqliteDbError(e) } +} + +#[cfg(target_arch = "wasm32")] +impl From for GetTxDetailsError { + fn from(e: String) -> Self { GetTxDetailsError::IndexedDbError(e) } +} + +impl From> for GetTxDetailsError { + fn from(e: MmError) -> Self { GetTxDetailsError::Internal(e.into_inner()) } +} + +// This is how the error is stringified before sending it to the client. +impl ToString for GetTxDetailsError { + fn to_string(&self) -> String { + match self { + GetTxDetailsError::UtxoRpcError(e) => format!("RPC Error: {e:?}"), + #[cfg(not(target_arch = "wasm32"))] + GetTxDetailsError::SqliteDbError(e) => format!("Sqlite DB Error: {e:?}"), + #[cfg(target_arch = "wasm32")] + GetTxDetailsError::IndexedDbError(e) => format!("IndexedDB Error: {e:?}"), + GetTxDetailsError::Internal(e) => format!("Internal Error: {e:?}"), + } + } +} + +async fn get_tx_details(coin: &ZCoin, txs: Vec>) -> Result, GetTxDetailsError> { + let current_block = coin.utxo_rpc_client().get_block_count().compat().await?; + let tx_ids = txs.iter().map(|tx| tx.txid).collect(); + let txs_from_db = fetch_txs_from_db(coin, tx_ids).await?; + + let hashes_for_verbose = txs_from_db + .iter() + .map(|item| H256Json::from(item.tx_hash.as_slice())) + .collect(); + let transactions = coin.z_transactions_from_cache_or_rpc(hashes_for_verbose).await?; + + let prev_tx_hashes = transactions + .iter() + .flat_map(|(_, tx)| { + tx.vin.iter().map(|vin| { + let mut hash = *vin.prevout.hash(); + hash.reverse(); + H256Json::from(hash) + }) + }) + .collect(); + let prev_transactions = coin.z_transactions_from_cache_or_rpc(prev_tx_hashes).await?; + + let txs_details = txs_from_db + .into_iter() + .map(|sql_item| coin.tx_details_from_db_item(sql_item, &transactions, &prev_transactions, current_block)) + .collect::>()?; + + Ok(txs_details) +} diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 5909b5cb42..2b29f093ed 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -20,13 +20,15 @@ pub struct ZCoinBalanceEventStreamer { impl ZCoinBalanceEventStreamer { pub fn new(coin: ZCoin) -> Self { Self { coin } } + + pub fn derive_streamer_id(coin: &str) -> String { format!("BALANCE:{coin}") } } #[async_trait] impl EventStreamer for ZCoinBalanceEventStreamer { type DataInType = (); - fn streamer_id(&self) -> String { format!("BALANCE:{}", self.coin.ticker()) } + fn streamer_id(&self) -> String { Self::derive_streamer_id(self.coin.ticker()) } async fn handle( self, diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 7fcf06cb12..55e9ccc633 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -301,6 +301,7 @@ impl From for ZTxHistoryError { fn from(err: CursorError) -> Self { ZTxHistoryError::IndexedDbError(err.to_string()) } } +#[derive(Debug)] pub(super) struct NoInfoAboutTx(pub(super) H256Json); impl From for MyTxHistoryErrorV2 { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index bd71d554c6..61f3d6a291 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -17,6 +17,7 @@ use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; use hex::{FromHex, FromHexError}; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use parking_lot::Mutex; use prost::Message; use rpc::v1::types::{Bytes, H256 as H256Json}; @@ -33,7 +34,6 @@ use zcash_primitives::zip32::ExtendedSpendingKey; pub(crate) mod z_coin_grpc { tonic::include_proto!("pirate.wallet.sdk.rpc"); } -use crate::z_coin::z_balance_streaming::ZBalanceEventSender; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{ChainSpec, CompactBlock as TonicCompactBlock}; @@ -509,7 +509,6 @@ pub(super) async fn init_light_client<'a>( sync_params: &Option, skip_sync_params: bool, z_spending_key: &ExtendedSpendingKey, - z_balance_event_sender: Option, ) -> Result<(AsyncMutex, WalletDbShared), MmError> { let coin = builder.ticker.to_string(); let (sync_status_notifier, sync_watcher) = channel(1); @@ -571,7 +570,7 @@ pub(super) async fn init_light_client<'a>( scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration, scan_interval_ms: builder.z_coin_params.scan_interval_ms, first_sync_block: first_sync_block.clone(), - z_balance_event_sender, + streaming_manager: builder.ctx.event_stream_manager.clone(), }; let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(light_rpc_clients))); @@ -588,7 +587,6 @@ pub(super) async fn init_native_client<'a>( native_client: NativeClient, blocks_db: BlockDbImpl, z_spending_key: &ExtendedSpendingKey, - z_balance_event_sender: Option, ) -> Result<(AsyncMutex, WalletDbShared), MmError> { let coin = builder.ticker.to_string(); let (sync_status_notifier, sync_watcher) = channel(1); @@ -619,7 +617,7 @@ pub(super) async fn init_native_client<'a>( scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration, scan_interval_ms: builder.z_coin_params.scan_interval_ms, first_sync_block: first_sync_block.clone(), - z_balance_event_sender, + streaming_manager: builder.ctx.event_stream_manager.clone(), }; let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(native_client))); @@ -718,7 +716,8 @@ pub struct SaplingSyncLoopHandle { scan_blocks_per_iteration: u32, scan_interval_ms: u64, first_sync_block: FirstSyncBlock, - z_balance_event_sender: Option, + /// A copy of the streaming manager to send notifications to the streamers upon new txs, balance change, etc... + streaming_manager: StreamingManager, } impl SaplingSyncLoopHandle { @@ -842,7 +841,7 @@ impl SaplingSyncLoopHandle { blocks_db .process_blocks_with_mode( self.consensus_params.clone(), - BlockProcessingMode::Scan(scan, self.z_balance_event_sender.clone()), + BlockProcessingMode::Scan(scan, self.streaming_manager.clone()), None, Some(self.scan_blocks_per_iteration), ) diff --git a/mm2src/coins/z_coin/z_tx_history.rs b/mm2src/coins/z_coin/z_tx_history.rs index 57eb2fdb4c..b9f1c772cf 100644 --- a/mm2src/coins/z_coin/z_tx_history.rs +++ b/mm2src/coins/z_coin/z_tx_history.rs @@ -1,6 +1,9 @@ +use std::collections::HashSet; + use crate::z_coin::{ZCoin, ZTxHistoryError}; use common::PagingOptionsEnum; use mm2_err_handle::prelude::MmError; +use zcash_primitives::transaction::TxId; cfg_wasm32!( use crate::z_coin::storage::wasm::tables::{WalletDbBlocksTable, WalletDbReceivedNotesTable, WalletDbTransactionsTable}; @@ -220,3 +223,16 @@ pub(crate) async fn fetch_tx_history_from_db( }) .await } + +#[cfg(target_arch = "wasm32")] +pub(crate) async fn fetch_txs_from_db(_z: &ZCoin, _tx_ids: HashSet) -> Result, String> { + unimplemented!() +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) async fn fetch_txs_from_db( + _z: &ZCoin, + _tx_ids: HashSet, +) -> Result, SqliteError> { + unimplemented!() +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs index a8657a8f86..cafa69d686 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs @@ -2,13 +2,13 @@ use super::EnableStreamingResponse; use coins::utxo::tx_history_events::TxHistoryEventStreamer; +use coins::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; use coins::{lp_coinfind, MmCoin, MmCoinEnum}; use common::HttpStatusCode; use http::StatusCode; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; - #[derive(Deserialize)] pub struct EnableTxHistoryStreamingRequest { pub client_id: u64, @@ -71,12 +71,12 @@ pub async fn enable_tx_history( .add(req.client_id, streamer, coin.spawner()) .await }, - // MmCoinEnum::ZCoin(coin) => { - // let streamer = ZCoinTxHistoryEventStreamer::new(req.coin); - // ctx.event_stream_manager - // .add(req.client_id, streamer, coin.spawner()) - // .await - // }, + MmCoinEnum::ZCoin(coin) => { + let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); + ctx.event_stream_manager + .add(req.client_id, streamer, coin.spawner()) + .await + }, // FIXME: What about tokens?! _ => Err(TxHistoryStreamingRequestError::CoinNotSupported)?, }; From 3269faa1a9873985b52460244985270a654065e3 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 13 Aug 2024 16:14:14 +0200 Subject: [PATCH 078/128] review: suggestions by sami 1 --- mm2src/coins/z_coin/tx_history_events.rs | 31 +++++++-------- .../src/rpc/streaming_activations/balance.rs | 39 +++++++------------ 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/mm2src/coins/z_coin/tx_history_events.rs b/mm2src/coins/z_coin/tx_history_events.rs index 22763b0a4d..a84ed820cb 100644 --- a/mm2src/coins/z_coin/tx_history_events.rs +++ b/mm2src/coins/z_coin/tx_history_events.rs @@ -57,13 +57,20 @@ impl EventStreamer for ZCoinTxHistoryEventStreamer { } } -#[derive(Debug)] +/// Errors that can occur while getting transaction details for some tx hashes. +/// +/// The error implements `Display` trait, so it can be easily converted `.to_string`. +#[derive(Debug, derive_more::Display)] enum GetTxDetailsError { + #[display(fmt = "RPC Error: {_0:?}")] UtxoRpcError(UtxoRpcError), #[cfg(not(target_arch = "wasm32"))] + #[display(fmt = "Sqlite DB Error: {_0:?}")] SqliteDbError(db_common::sqlite::rusqlite::Error), #[cfg(target_arch = "wasm32")] + #[display(fmt = "IndexedDB Error: {_0:?}")] IndexedDbError(String), + #[display(fmt = "Internal Error: {_0:?}")] Internal(NoInfoAboutTx), } @@ -85,24 +92,12 @@ impl From> for GetTxDetailsError { fn from(e: MmError) -> Self { GetTxDetailsError::Internal(e.into_inner()) } } -// This is how the error is stringified before sending it to the client. -impl ToString for GetTxDetailsError { - fn to_string(&self) -> String { - match self { - GetTxDetailsError::UtxoRpcError(e) => format!("RPC Error: {e:?}"), - #[cfg(not(target_arch = "wasm32"))] - GetTxDetailsError::SqliteDbError(e) => format!("Sqlite DB Error: {e:?}"), - #[cfg(target_arch = "wasm32")] - GetTxDetailsError::IndexedDbError(e) => format!("IndexedDB Error: {e:?}"), - GetTxDetailsError::Internal(e) => format!("Internal Error: {e:?}"), - } - } -} - async fn get_tx_details(coin: &ZCoin, txs: Vec>) -> Result, GetTxDetailsError> { let current_block = coin.utxo_rpc_client().get_block_count().compat().await?; - let tx_ids = txs.iter().map(|tx| tx.txid).collect(); - let txs_from_db = fetch_txs_from_db(coin, tx_ids).await?; + let txs_from_db = { + let tx_ids = txs.iter().map(|tx| tx.txid).collect(); + fetch_txs_from_db(coin, tx_ids).await? + }; let hashes_for_verbose = txs_from_db .iter() @@ -124,7 +119,7 @@ async fn get_tx_details(coin: &ZCoin, txs: Vec>) -> Result>()?; Ok(txs_details) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index b938bfe217..5f34539f92 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -51,33 +51,21 @@ pub async fn enable_balance( let enable_result = match coin { MmCoinEnum::UtxoCoin(coin) => { - if req.config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } + check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager .add(req.client_id, streamer, coin.spawner()) .await }, MmCoinEnum::Bch(coin) => { - if req.config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } + check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager .add(req.client_id, streamer, coin.spawner()) .await }, MmCoinEnum::QtumCoin(coin) => { - if req.config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } + check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager .add(req.client_id, streamer, coin.spawner()) @@ -91,22 +79,14 @@ pub async fn enable_balance( .await }, MmCoinEnum::ZCoin(coin) => { - if req.config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } + check_empty_config(&req.config)?; let streamer = ZCoinBalanceEventStreamer::new(coin.clone()); ctx.event_stream_manager .add(req.client_id, streamer, coin.spawner()) .await }, MmCoinEnum::Tendermint(coin) => { - if req.config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } + check_empty_config(&req.config)?; let streamer = TendermintBalanceEventStreamer::new(coin.clone()); ctx.event_stream_manager .add(req.client_id, streamer, coin.spawner()) @@ -120,3 +100,12 @@ pub async fn enable_balance( .map(EnableStreamingResponse::new) .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}"))) } + +fn check_empty_config(config: &Option) -> MmResult<(), BalanceStreamingRequestError> { + if config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + Ok(()) +} From 1624a3ec71f64f71a250e91d3b190516c30f3d0e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 13 Aug 2024 17:30:35 +0200 Subject: [PATCH 079/128] streaming_manager: manage streamers & clients & controller from one lock --- mm2src/mm2_event_stream/src/manager.rs | 125 +++++++++++-------------- 1 file changed, 57 insertions(+), 68 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 9538637c91..79bfa67ffb 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,6 +1,7 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex}; +use std::ops::Deref; +use std::sync::Arc; use crate::{controller::Controller, Broadcaster, Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; @@ -51,21 +52,20 @@ impl StreamerInfo { fn is_down(&self) -> bool { self.shutdown.is_canceled() } } -#[derive(Clone, Default)] -pub struct StreamingManager { +#[derive(Default)] +struct StreamingManagerInner { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. - // Using an RwLock assuming `StreamingManager::send` will be heavily used. - streamers: Arc>>, + streamers: HashMap, /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). - // Using an RwLock assuming `StreamingManager::broadcast` will be heavily used. - controller: Arc>>, + controller: Controller, /// An inverse map from clients to the streamers they are listening to. /// Lists known clients and is always kept in sync with `streamers`. - // NOTE: Any operation writing to this field will also write to `streamers` and potentially `controller`. - // Make sure to lock this field first and all the way until the changes are made to other fields. - clients: Arc>>>, + clients: HashMap>, } +#[derive(Clone, Default)] +pub struct StreamingManager(Arc>); + impl StreamingManager { /// Spawns and adds a new streamer `streamer` to the manager. pub async fn add( @@ -78,11 +78,10 @@ impl StreamingManager { // Remove the streamer if it died for some reason. self.remove_streamer_if_down(&streamer_id); - // Pre-checks before spawning the streamer. Inside another scope to drop the locks early. + // Pre-checks before spawning the streamer. Inside another scope to drop the lock early. { - let mut clients = self.clients.lock().unwrap(); - - match clients.get(&client_id) { + let mut this = self.0.write(); + match this.clients.get(&client_id) { // We don't know that client. We don't have a connection to it. None => return Err(StreamingManagerError::UnknownClient), // The client is already listening to that streamer. @@ -93,11 +92,11 @@ impl StreamingManager { } // If a streamer is already up and running, we won't spawn another one. - if let Some(streamer_info) = self.streamers.write().get_mut(&streamer_id) { + if let Some(streamer_info) = this.streamers.get_mut(&streamer_id) { // Register the client as a listener to the streamer. streamer_info.clients.insert(client_id); // Register the streamer as listened-to by the client. - clients + this.clients .get_mut(&client_id) .map(|listening_to| listening_to.insert(streamer_id.clone())); return Ok(streamer_id); @@ -111,14 +110,13 @@ impl StreamingManager { .map_err(StreamingManagerError::SpawnError)?; let streamer_info = StreamerInfo::new(data_in, shutdown); - // Note that we didn't hold the locks while spawning the streamer (potentially a long operation). + // Note that we didn't hold the lock while spawning the streamer (potentially a long operation). // This means we can't assume either that the client still exists at this point or // that the streamer still doesn't exist. - let mut clients = self.clients.lock().unwrap(); - if let Some(listening_to) = clients.get_mut(&client_id) { + let mut this = self.0.write(); + if let Some(listening_to) = this.clients.get_mut(&client_id) { listening_to.insert(streamer_id.clone()); - self.streamers - .write() + this.streamers .entry(streamer_id.clone()) .or_insert(streamer_info) .clients @@ -133,8 +131,9 @@ impl StreamingManager { /// Sends data to a streamer with `streamer_id`. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { - let streamers = self.streamers.read(); - let streamer_info = streamers + let this = self.0.read(); + let streamer_info = this + .streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; @@ -153,8 +152,9 @@ impl StreamingManager { streamer_id: &str, data_fn: impl FnOnce() -> T, ) -> Result<(), StreamingManagerError> { - let streamers = self.streamers.read(); - let streamer_info = streamers + let this = self.0.read(); + let streamer_info = this + .streamers .get(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)?; let data_in = streamer_info.data_in.as_ref().ok_or(StreamingManagerError::NoDataIn)?; @@ -165,20 +165,19 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { - let mut clients = self.clients.lock().unwrap(); - if let Some(listening_to) = clients.get_mut(&client_id) { + let mut this = self.0.write(); + if let Some(listening_to) = this.clients.get_mut(&client_id) { listening_to.remove(streamer_id); - let mut streamers = self.streamers.write(); - streamers + this.streamers .get_mut(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)? .clients .remove(&client_id); // If there are no more listening clients, terminate the streamer. - if streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { - streamers.remove(streamer_id); + if this.streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { + this.streamers.remove(streamer_id); } Ok(()) } else { @@ -192,33 +191,36 @@ impl StreamingManager { /// this method broadcasts an event to the listening *clients* directly, independently /// of any streamer (i.e. bypassing any streamer). pub fn broadcast(&self, event: Event) { - if let Some(client_ids) = self.streamers.read().get(event.origin()).map(|info| &info.clients) { - self.controller.read().broadcast(event, Some(client_ids)) + let this = self.0.read(); + if let Some(client_ids) = this.streamers.get(event.origin()).map(|info| &info.clients) { + this.controller.broadcast(event, Some(client_ids)) } } /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. - pub fn broadcast_all(&self, event: Event) { self.controller.read().broadcast(event, None) } + pub fn broadcast_all(&self, event: Event) { self.0.read().controller.broadcast(event, None) } pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { - let mut clients = self.clients.lock().unwrap(); - if clients.contains_key(&client_id) { + let mut this = self.0.write(); + if this.clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists); } - clients.insert(client_id, HashSet::new()); + this.clients.insert(client_id, HashSet::new()); // Note that events queued in the channel are `Arc<` shared. // So a 1024 long buffer isn't actually heavy on memory. - Ok(self.controller.write().create_channel(client_id, 1024)) + Ok(this.controller.create_channel(client_id, 1024)) } pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { - let mut clients = self.clients.lock().unwrap(); + let mut this = self.0.write(); // Remove the client from our known-clients map. - let listening_to = clients.remove(&client_id).ok_or(StreamingManagerError::UnknownClient)?; + let listening_to = this + .clients + .remove(&client_id) + .ok_or(StreamingManagerError::UnknownClient)?; // Remove the client from all the streamers it was listening to. - let mut streamers = self.streamers.write(); for streamer_id in listening_to { - if let Some(streamer_info) = streamers.get_mut(&streamer_id) { + if let Some(streamer_info) = this.streamers.get_mut(&streamer_id) { streamer_info.clients.remove(&client_id); } else { log::error!( @@ -226,12 +228,12 @@ impl StreamingManager { ); } // If there are no more listening clients, terminate the streamer. - if streamers.get(&streamer_id).map(|info| info.clients.len()) == Some(0) { - streamers.remove(&streamer_id); + if this.streamers.get(&streamer_id).map(|info| info.clients.len()) == Some(0) { + this.streamers.remove(&streamer_id); } } // Remove our channel with this client. - self.controller.write().remove_channel(&client_id); + this.controller.remove_channel(&client_id); Ok(()) } @@ -241,18 +243,17 @@ impl StreamingManager { /// the streamer might die by itself (e.g. the spawner it was spawned with aborted). /// In this case, we need to remove the streamer and de-list it from all clients. fn remove_streamer_if_down(&self, streamer_id: &str) { - let mut clients = self.clients.lock().unwrap(); - let mut streamers = self.streamers.write(); - if let Some(streamer_info) = streamers.get(streamer_id) { + let mut this = self.0.write(); + if let Some(streamer_info) = this.streamers.get_mut(streamer_id) { if streamer_info.is_down() { // Remove the streamer from all clients listening to it. - for client_id in &streamer_info.clients { - clients - .get_mut(client_id) + for client_id in std::mem::take(&mut streamer_info.clients) { + this.clients + .get_mut(&client_id) .map(|listening_to| listening_to.remove(streamer_id)); } // And remove the streamer from our registry. - streamers.remove(streamer_id); + this.streamers.remove(streamer_id); } } } @@ -414,21 +415,15 @@ mod tests { // The streamer is up and streaming to `client_id`. assert!(manager - .streamers .read() + .streamers .get(&streamer_id) .unwrap() .clients .contains(&client_id)); // The client should be registered and listening to `streamer_id`. - assert!(manager - .clients - .lock() - .unwrap() - .get(&client_id) - .unwrap() - .contains(&streamer_id)); + assert!(manager.read().clients.get(&client_id).unwrap().contains(&streamer_id)); // Abort the system to kill the streamer. system.abort_all().unwrap(); @@ -438,14 +433,8 @@ mod tests { manager.remove_streamer_if_down(&streamer_id); // The streamer should be removed. - assert!(manager.streamers.read().get(&streamer_id).is_none()); + assert!(manager.read().streamers.get(&streamer_id).is_none()); // And the client is no more listening to it. - assert!(!manager - .clients - .lock() - .unwrap() - .get(&client_id) - .unwrap() - .contains(&streamer_id)); + assert!(!manager.read().clients.get(&client_id).unwrap().contains(&streamer_id)); } } From a8cd6a65b34a5e72408ce0af8c3e500f62a66d23 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 13 Aug 2024 18:21:00 +0200 Subject: [PATCH 080/128] get rid of the controller inlined the controller logic right inside the streaming manager. it was simple enough to have it's own struct + having it essentially replicated the clients hashmap for no reason --- mm2src/mm2_event_stream/src/controller.rs | 130 --------------------- mm2src/mm2_event_stream/src/lib.rs | 1 - mm2src/mm2_event_stream/src/manager.rs | 133 +++++++++++++++------- mm2src/mm2_event_stream/src/streamer.rs | 13 +-- 4 files changed, 97 insertions(+), 180 deletions(-) delete mode 100644 mm2src/mm2_event_stream/src/controller.rs diff --git a/mm2src/mm2_event_stream/src/controller.rs b/mm2src/mm2_event_stream/src/controller.rs deleted file mode 100644 index 4de601fb9c..0000000000 --- a/mm2src/mm2_event_stream/src/controller.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::{collections::{HashMap, HashSet}, - sync::Arc}; -use tokio::sync::mpsc::{self, Receiver, Sender}; - -#[derive(Clone, Default)] -/// Root controller of streaming channels -pub struct Controller { - channels: HashMap>>, -} - -impl Controller { - /// Creates a new channel and returns it's events receiver - pub(crate) fn create_channel(&mut self, channel_id: u64, concurrency: usize) -> Receiver> { - let (tx, rx) = mpsc::channel(concurrency); - self.channels.insert(channel_id, tx); - rx - } - - /// Removes the channel from the controller - pub(crate) fn remove_channel(&mut self, channel_id: &u64) { self.channels.remove(channel_id); } - - /// Broadcast message to all channels - pub(crate) fn broadcast(&self, message: M, client_ids: Option<&HashSet>) { - let msg = Arc::new(message); - for tx in self.channels(client_ids) { - // Only `try_send` here. If the receiver's channel is full (receiver is slow), it will - // not receive the message. This avoids blocking the broadcast to other receivers. - tx.try_send(msg.clone()).ok(); - } - } - - /// Returns the channels that are associated with the specified client ids. - /// - /// If no client ids are specified, all the channels are returned. - fn channels(&self, client_ids: Option<&HashSet>) -> Vec>> { - if let Some(client_ids) = client_ids { - self.channels - .iter() - .filter_map(|(id, sender)| client_ids.contains(id).then_some(sender)) - .cloned() - .collect() - } else { - // Returns all the channels if no client ids where specifically requested. - self.channels.values().cloned().collect() - } - } -} - -#[cfg(any(test, target_arch = "wasm32"))] -mod tests { - use super::*; - use common::cross_test; - use tokio::sync::mpsc::error::TryRecvError; - - common::cfg_wasm32! { - use wasm_bindgen_test::*; - wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); - } - - cross_test!(test_create_channel_and_broadcast, { - let mut controller = Controller::default(); - let mut channel_receiver = controller.create_channel(0, 1); - - controller.broadcast("Message".to_string(), None); - - let received_msg = channel_receiver.recv().await.unwrap(); - assert_eq!(*received_msg, "Message".to_string()); - }); - - cross_test!(test_multiple_channels_and_broadcast, { - let mut controller = Controller::default(); - - let mut receivers = Vec::new(); - for channel_id in 0..3 { - receivers.push(controller.create_channel(channel_id, 1)); - } - - controller.broadcast("Message".to_string(), None); - - for receiver in &mut receivers { - let received_msg = receiver.recv().await.unwrap(); - assert_eq!(*received_msg, "Message".to_string()); - } - }); - - cross_test!(test_broadcast_across_channels, { - let mut controller = Controller::default(); - - let mut receivers = Vec::new(); - for channel_id in 0..3 { - receivers.push(controller.create_channel(channel_id, 1)); - } - - controller.broadcast("Message".to_string(), None); - - for receiver in &mut receivers { - let received_msg = receiver.recv().await.unwrap(); - assert_eq!(*received_msg, "Message".to_string()); - } - }); - - cross_test!(test_msgs_drop_past_capacity, { - let mut controller = Controller::default(); - let mut channel_receiver = controller.create_channel(0, 6); - - controller.broadcast("Message 1".to_string(), None); - controller.broadcast("Message 2".to_string(), None); - controller.broadcast("Message 3".to_string(), None); - controller.broadcast("Message 4".to_string(), None); - controller.broadcast("Message 5".to_string(), None); - controller.broadcast("Message 6".to_string(), None); - // This message should be dropped. - controller.broadcast("Message 7".to_string(), None); - - let mut received_msgs = Vec::new(); - for _ in 0..6 { - let received_msg = channel_receiver.recv().await.unwrap(); - received_msgs.push(received_msg); - } - - assert_eq!(*received_msgs[0], "Message 1".to_string()); - assert_eq!(*received_msgs[1], "Message 2".to_string()); - assert_eq!(*received_msgs[2], "Message 3".to_string()); - assert_eq!(*received_msgs[3], "Message 4".to_string()); - assert_eq!(*received_msgs[4], "Message 5".to_string()); - assert_eq!(*received_msgs[5], "Message 6".to_string()); - - assert!(matches!(channel_receiver.try_recv(), Err(TryRecvError::Empty))); - }); -} diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index 5137d0953c..f60d2a9d4a 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,4 +1,3 @@ -mod controller; pub mod event; pub mod manager; pub mod streamer; diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 79bfa67ffb..b8f08e0ad2 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,16 +1,15 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; -use std::ops::Deref; use std::sync::Arc; -use crate::{controller::Controller, Broadcaster, Event, EventStreamer}; +use crate::{Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; use parking_lot::RwLock; -use tokio::sync::mpsc::Receiver; +use tokio::sync::mpsc; /// The errors that could originate from the streaming manager. #[derive(Debug)] @@ -49,18 +48,48 @@ impl StreamerInfo { } } + fn add_client(&mut self, client_id: u64) { self.clients.insert(client_id); } + + fn remove_client(&mut self, client_id: &u64) { self.clients.remove(client_id); } + fn is_down(&self) -> bool { self.shutdown.is_canceled() } } +struct ClientInfo { + /// The streamers the client is listening to. + listening_to: HashSet, + /// The communication/stream-out channel to the client. + channel: mpsc::Sender>, +} + +impl ClientInfo { + fn new(channel: mpsc::Sender>) -> Self { + Self { + listening_to: HashSet::new(), + channel, + } + } + + fn add_streamer(&mut self, streamer_id: String) { self.listening_to.insert(streamer_id); } + + fn remove_streamer(&mut self, streamer_id: &str) { self.listening_to.remove(streamer_id); } + + fn listens_to(&self, streamer_id: &str) -> bool { self.listening_to.contains(streamer_id) } + + fn send_event(&self, event: Arc) { + // Only `try_send` here. If the channel is full (client is slow), the message + // will be dropped and the client won't receive it. + // This avoids blocking the broadcast to other receivers. + self.channel.try_send(event).ok(); + } +} + #[derive(Default)] struct StreamingManagerInner { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: HashMap, - /// The stream-out controller/broadcaster that all streamers use to stream data to the clients (using SSE). - controller: Controller, - /// An inverse map from clients to the streamers they are listening to. - /// Lists known clients and is always kept in sync with `streamers`. - clients: HashMap>, + /// An inverse map from client IDs to the streamers they are listening to and the communication channel with the client. + clients: HashMap, } #[derive(Clone, Default)] @@ -85,7 +114,7 @@ impl StreamingManager { // We don't know that client. We don't have a connection to it. None => return Err(StreamingManagerError::UnknownClient), // The client is already listening to that streamer. - Some(listening_to) if listening_to.contains(&streamer_id) => { + Some(client_info) if client_info.listens_to(&streamer_id) => { return Err(StreamingManagerError::ClientAlreadyListening); }, _ => (), @@ -94,18 +123,18 @@ impl StreamingManager { // If a streamer is already up and running, we won't spawn another one. if let Some(streamer_info) = this.streamers.get_mut(&streamer_id) { // Register the client as a listener to the streamer. - streamer_info.clients.insert(client_id); + streamer_info.add_client(client_id); // Register the streamer as listened-to by the client. this.clients .get_mut(&client_id) - .map(|listening_to| listening_to.insert(streamer_id.clone())); + .map(|info| info.add_streamer(streamer_id.clone())); return Ok(streamer_id); } } // Spawn a new streamer. let (shutdown, data_in) = streamer - .spawn(spawner, Broadcaster::new(self.clone())) + .spawn(spawner, self.clone()) .await .map_err(StreamingManagerError::SpawnError)?; let streamer_info = StreamerInfo::new(data_in, shutdown); @@ -114,13 +143,12 @@ impl StreamingManager { // This means we can't assume either that the client still exists at this point or // that the streamer still doesn't exist. let mut this = self.0.write(); - if let Some(listening_to) = this.clients.get_mut(&client_id) { - listening_to.insert(streamer_id.clone()); + if let Some(client_info) = this.clients.get_mut(&client_id) { + client_info.add_streamer(streamer_id.clone()); this.streamers .entry(streamer_id.clone()) .or_insert(streamer_info) - .clients - .insert(client_id); + .add_client(client_id); } else { // The client was removed while we were spawning the streamer. // We no longer have a connection for it. @@ -166,14 +194,13 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { let mut this = self.0.write(); - if let Some(listening_to) = this.clients.get_mut(&client_id) { - listening_to.remove(streamer_id); + if let Some(client_info) = this.clients.get_mut(&client_id) { + client_info.remove_streamer(streamer_id); this.streamers .get_mut(streamer_id) .ok_or(StreamingManagerError::StreamerNotFound)? - .clients - .remove(&client_id); + .remove_client(&client_id); // If there are no more listening clients, terminate the streamer. if this.streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { @@ -191,37 +218,49 @@ impl StreamingManager { /// this method broadcasts an event to the listening *clients* directly, independently /// of any streamer (i.e. bypassing any streamer). pub fn broadcast(&self, event: Event) { + let event = Arc::new(event); let this = self.0.read(); if let Some(client_ids) = this.streamers.get(event.origin()).map(|info| &info.clients) { - this.controller.broadcast(event, Some(client_ids)) - } + client_ids.iter().for_each(|client_id| { + this.clients.get(client_id).map(|info| info.send_event(event.clone())); + }); + }; } /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. - pub fn broadcast_all(&self, event: Event) { self.0.read().controller.broadcast(event, None) } + pub fn broadcast_all(&self, event: Event) { + let event = Arc::new(event); + self.0.read().clients.values().for_each(|info| { + info.send_event(event.clone()); + }); + } - pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { + /// Creates a new client and returns the event receiver for this client. + pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { let mut this = self.0.write(); if this.clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists); } - this.clients.insert(client_id, HashSet::new()); // Note that events queued in the channel are `Arc<` shared. // So a 1024 long buffer isn't actually heavy on memory. - Ok(this.controller.create_channel(client_id, 1024)) + let (tx, rx) = mpsc::channel(1024); + let client_info = ClientInfo::new(tx); + this.clients.insert(client_id, client_info); + Ok(rx) } + /// Removes a client from the manager. pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { let mut this = self.0.write(); // Remove the client from our known-clients map. - let listening_to = this + let client_info = this .clients .remove(&client_id) .ok_or(StreamingManagerError::UnknownClient)?; // Remove the client from all the streamers it was listening to. - for streamer_id in listening_to { + for streamer_id in client_info.listening_to { if let Some(streamer_info) = this.streamers.get_mut(&streamer_id) { - streamer_info.clients.remove(&client_id); + streamer_info.remove_client(&client_id); } else { log::error!( "Client {client_id} was listening to a non-existent streamer {streamer_id}. This is a bug!" @@ -232,8 +271,6 @@ impl StreamingManager { this.streamers.remove(&streamer_id); } } - // Remove our channel with this client. - this.controller.remove_channel(&client_id); Ok(()) } @@ -246,14 +283,15 @@ impl StreamingManager { let mut this = self.0.write(); if let Some(streamer_info) = this.streamers.get_mut(streamer_id) { if streamer_info.is_down() { - // Remove the streamer from all clients listening to it. - for client_id in std::mem::take(&mut streamer_info.clients) { - this.clients - .get_mut(&client_id) - .map(|listening_to| listening_to.remove(streamer_id)); + // Remove the streamer from our registry. + if let Some(streamer_info) = this.streamers.remove(streamer_id) { + // And remove the streamer from all clients listening to it. + for client_id in streamer_info.clients { + this.clients + .get_mut(&client_id) + .map(|info| info.remove_streamer(streamer_id)); + } } - // And remove the streamer from our registry. - this.streamers.remove(streamer_id); } } } @@ -415,6 +453,7 @@ mod tests { // The streamer is up and streaming to `client_id`. assert!(manager + .0 .read() .streamers .get(&streamer_id) @@ -423,7 +462,13 @@ mod tests { .contains(&client_id)); // The client should be registered and listening to `streamer_id`. - assert!(manager.read().clients.get(&client_id).unwrap().contains(&streamer_id)); + assert!(manager + .0 + .read() + .clients + .get(&client_id) + .unwrap() + .listens_to(&streamer_id)); // Abort the system to kill the streamer. system.abort_all().unwrap(); @@ -433,8 +478,14 @@ mod tests { manager.remove_streamer_if_down(&streamer_id); // The streamer should be removed. - assert!(manager.read().streamers.get(&streamer_id).is_none()); + assert!(manager.0.read().streamers.get(&streamer_id).is_none()); // And the client is no more listening to it. - assert!(!manager.read().clients.get(&client_id).unwrap().contains(&streamer_id)); + assert!(!manager + .0 + .read() + .clients + .get(&client_id) + .unwrap() + .listens_to(&streamer_id)); } } diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index d552749158..d41ed920a9 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -47,7 +47,7 @@ where async fn spawn( self, spawner: WeakSpawner, - broadcaster: Broadcaster, + streaming_manager: StreamingManager, ) -> Result<(oneshot::Sender<()>, Option>>), String> { let streamer_id = self.streamer_id(); info!("Spawning event streamer: {streamer_id}"); @@ -81,7 +81,7 @@ where _ = rx_shutdown.fuse() => { info!("Manually shutting down event streamer: {streamer_id}.") } - _ = self.handle(broadcaster, tx_ready, data_receiver).fuse() => {} + _ = self.handle(Broadcaster::new(streaming_manager), tx_ready, data_receiver).fuse() => {} } } }; @@ -197,10 +197,9 @@ mod tests { #[tokio::test] async fn test_spawn_periodic_streamer() { let system = AbortableQueue::default(); - let broadcaster = Broadcaster::new(StreamingManager::default()); // Spawn the periodic streamer. let (_, data_in) = PeriodicStreamer - .spawn(system.weak_spawner(), broadcaster) + .spawn(system.weak_spawner(), StreamingManager::default()) .await .unwrap(); // Periodic streamer shouldn't be ingesting any input. @@ -210,10 +209,9 @@ mod tests { #[tokio::test] async fn test_spawn_reactive_streamer() { let system = AbortableQueue::default(); - let broadcaster = Broadcaster::new(StreamingManager::default()); // Spawn the reactive streamer. let (_, data_in) = ReactiveStreamer - .spawn(system.weak_spawner(), broadcaster) + .spawn(system.weak_spawner(), StreamingManager::default()) .await .unwrap(); // Reactive streamer should be ingesting some input. @@ -223,10 +221,9 @@ mod tests { #[tokio::test] async fn test_spawn_erroring_streamer() { let system = AbortableQueue::default(); - let broadcaster = Broadcaster::new(StreamingManager::default()); // Try to spawn the erroring streamer. let err = InitErrorStreamer - .spawn(system.weak_spawner(), broadcaster) + .spawn(system.weak_spawner(), StreamingManager::default()) .await .unwrap_err(); // The streamer should return an error. From 896636df056dfc6aeeccd2c685cd59f70ec481d0 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 13 Aug 2024 18:58:39 +0200 Subject: [PATCH 081/128] explain why tokio is used instead of futures for the channel with clients --- mm2src/mm2_event_stream/Cargo.toml | 4 ++-- mm2src/mm2_event_stream/src/manager.rs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_event_stream/Cargo.toml b/mm2src/mm2_event_stream/Cargo.toml index 369a22197d..5b1677fa0e 100644 --- a/mm2src/mm2_event_stream/Cargo.toml +++ b/mm2src/mm2_event_stream/Cargo.toml @@ -11,10 +11,10 @@ futures = { version = "0.3", default-features = false } parking_lot = "0.12" serde = { version = "1", features = ["derive", "rc"] } serde_json = { version = "1", features = ["preserve_order", "raw_value"] } -tokio = { version = "1", features = ["sync"] } +tokio = "1.20" [dev-dependencies] -tokio = { version = "1", features = ["sync", "macros", "time", "rt"] } +tokio = { version = "1.20", features = ["macros"] } [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-test = { version = "0.3.2" } diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index b8f08e0ad2..26eb532622 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -59,6 +59,11 @@ struct ClientInfo { /// The streamers the client is listening to. listening_to: HashSet, /// The communication/stream-out channel to the client. + // NOTE: Here we are using `tokio`'s `mpsc` because the one in `futures` have some extra feature + // (ref: https://users.rust-lang.org/t/why-does-try-send-from-crate-futures-require-mut-self/100389). + // This feature is aimed towards the multi-producer case (which we don't use) and requires a mutable + // reference on `try_send` calls. This will require us to put the channel in a mutex and degrade the + // broadcasting performance. channel: mpsc::Sender>, } From 1b8cab7701fd40a309fb7299617c2dfd5f5b9bef Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 8 Aug 2024 07:38:34 +0200 Subject: [PATCH 082/128] impl missing fetch_txs_from_db for zcoin --- mm2src/coins/z_coin/tx_history_events.rs | 20 +--- mm2src/coins/z_coin/z_tx_history.rs | 143 ++++++++++++++++++++++- 2 files changed, 142 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/z_coin/tx_history_events.rs b/mm2src/coins/z_coin/tx_history_events.rs index a84ed820cb..9097d581c7 100644 --- a/mm2src/coins/z_coin/tx_history_events.rs +++ b/mm2src/coins/z_coin/tx_history_events.rs @@ -1,5 +1,5 @@ use super::z_tx_history::fetch_txs_from_db; -use super::{NoInfoAboutTx, ZCoin, ZcoinTxDetails}; +use super::{NoInfoAboutTx, ZCoin, ZTxHistoryError, ZcoinTxDetails}; use crate::utxo::rpc_clients::UtxoRpcError; use crate::MarketCoinOps; use common::log; @@ -64,12 +64,8 @@ impl EventStreamer for ZCoinTxHistoryEventStreamer { enum GetTxDetailsError { #[display(fmt = "RPC Error: {_0:?}")] UtxoRpcError(UtxoRpcError), - #[cfg(not(target_arch = "wasm32"))] - #[display(fmt = "Sqlite DB Error: {_0:?}")] - SqliteDbError(db_common::sqlite::rusqlite::Error), - #[cfg(target_arch = "wasm32")] - #[display(fmt = "IndexedDB Error: {_0:?}")] - IndexedDbError(String), + #[display(fmt = "DB Error: {_0:?}")] + DbError(String), #[display(fmt = "Internal Error: {_0:?}")] Internal(NoInfoAboutTx), } @@ -78,14 +74,8 @@ impl From> for GetTxDetailsError { fn from(e: MmError) -> Self { GetTxDetailsError::UtxoRpcError(e.into_inner()) } } -#[cfg(not(target_arch = "wasm32"))] -impl From for GetTxDetailsError { - fn from(e: db_common::sqlite::rusqlite::Error) -> Self { GetTxDetailsError::SqliteDbError(e) } -} - -#[cfg(target_arch = "wasm32")] -impl From for GetTxDetailsError { - fn from(e: String) -> Self { GetTxDetailsError::IndexedDbError(e) } +impl From> for GetTxDetailsError { + fn from(e: MmError) -> Self { GetTxDetailsError::DbError(e.to_string()) } } impl From> for GetTxDetailsError { diff --git a/mm2src/coins/z_coin/z_tx_history.rs b/mm2src/coins/z_coin/z_tx_history.rs index b9f1c772cf..2fa6077ea3 100644 --- a/mm2src/coins/z_coin/z_tx_history.rs +++ b/mm2src/coins/z_coin/z_tx_history.rs @@ -9,6 +9,8 @@ cfg_wasm32!( use crate::z_coin::storage::wasm::tables::{WalletDbBlocksTable, WalletDbReceivedNotesTable, WalletDbTransactionsTable}; use crate::MarketCoinOps; use mm2_number::BigInt; + use mm2_db::indexed_db::cursor_prelude::CursorError; + use mm2_err_handle::prelude::MapToMmResult; use num_traits::ToPrimitive; ); @@ -225,14 +227,143 @@ pub(crate) async fn fetch_tx_history_from_db( } #[cfg(target_arch = "wasm32")] -pub(crate) async fn fetch_txs_from_db(_z: &ZCoin, _tx_ids: HashSet) -> Result, String> { - unimplemented!() +pub(crate) async fn fetch_txs_from_db( + z: &ZCoin, + tx_hashes: HashSet, +) -> Result, MmError> { + let wallet_db = z.z_fields.light_wallet_db.clone(); + let wallet_db = wallet_db.db.lock_db().await.unwrap(); + let db_transaction = wallet_db.get_inner().transaction().await?; + let tx_table = db_transaction.table::().await?; + + let limit = tx_hashes.len(); + let condition = { + // Convert TxIds to Vecs for comparison. + let tx_hashes: HashSet<_> = tx_hashes.into_iter().map(|txid| txid.0.to_vec()).collect(); + move |tx| { + let tx = serde_json::from_value::(tx) + .map_to_mm(|err| CursorError::ErrorDeserializingItem(err.to_string()))?; + Ok(tx_hashes.contains(&tx.txid)) + } + }; + + // Fetch transactions + let txs = tx_table + .cursor_builder() + .only("ticker", z.ticker())? + // We need to explicitly set a limit since `where_` implicitly sets a limit of 1 if no limit is set. + // TODO: Remove when `where_` doesn't exhibit this behavior. + .limit(limit) + .where_(condition) + .reverse() + .open_cursor("ticker") + .await? + .collect() + .await?; + + // Fetch received notes + let rn_table = db_transaction.table::().await?; + let received_notes = rn_table + .cursor_builder() + .only("ticker", z.ticker())? + .open_cursor("ticker") + .await? + .collect() + .await?; + + // Fetch blocks + let blocks_table = db_transaction.table::().await?; + let blocks = blocks_table + .cursor_builder() + .only("ticker", z.ticker())? + .open_cursor("ticker") + .await? + .collect() + .await?; + + // Process transactions and construct tx_details + let mut transactions = Vec::new(); + for (tx_id, tx) in txs { + if let Some((_, WalletDbBlocksTable { height, time, .. })) = blocks + .iter() + .find(|(_, block)| tx.block.map(|b| b == block.height).unwrap_or_default()) + { + let internal_id = tx_id; + let mut received_amount = 0; + let mut spent_amount = 0; + + for (_, note) in &received_notes { + if internal_id == note.tx { + received_amount += note.value.to_u64().ok_or_else(|| { + ZTxHistoryError::IndexedDbError("Number is too large to fit in a u64".to_string()) + })? as i64; + } + + // detecting spent amount by "spent" field in received_notes table + if let Some(spent) = ¬e.spent { + if &BigInt::from(internal_id) == spent { + spent_amount += note.value.to_u64().ok_or_else(|| { + ZTxHistoryError::IndexedDbError("Number is too large to fit in a u64".to_string()) + })? as i64; + } + } + } + + let mut tx_hash = tx.txid; + tx_hash.reverse(); + + transactions.push(ZCoinTxHistoryItem { + tx_hash, + internal_id: internal_id as i64, + height: *height as i64, + timestamp: *time as i64, + received_amount, + spent_amount, + }); + } + } + + Ok(transactions) } #[cfg(not(target_arch = "wasm32"))] pub(crate) async fn fetch_txs_from_db( - _z: &ZCoin, - _tx_ids: HashSet, -) -> Result, SqliteError> { - unimplemented!() + z: &ZCoin, + tx_hashes: HashSet, +) -> Result, MmError> { + let wallet_db = z.z_fields.light_wallet_db.clone(); + async_blocking(move || { + let sql_query = SqlBuilder::select_from(name!(TRANSACTIONS_TABLE; "txes")) + .field("txes.txid as tx_hash") + .field("txes.id_tx as internal_id") + .field("txes.block as block") + .field("blocks.time") + .field("COALESCE(rn.received_amount, 0)") + .field("COALESCE(sn.sent_amount, 0)") + .and_where_in("tx_hash", &tx_hashes.into_iter().collect::>()) + .left() + .join("(SELECT tx, SUM(value) as received_amount FROM received_notes GROUP BY tx) as rn") + .on("txes.id_tx = rn.tx") + .join("(SELECT spent, SUM(value) as sent_amount FROM received_notes GROUP BY spent) as sn") + .on("txes.id_tx = sn.spent") + .join(BLOCKS_TABLE) + .on("txes.block = blocks.height") + .group_by("internal_id") + .order_by("block", true) + .order_by("internal_id", false) + .sql() + .expect("valid query"); + + let txs = wallet_db + .db + .inner() + .lock() + .unwrap() + .sql_conn() + .prepare(&sql_query)? + .query_map([], ZCoinTxHistoryItem::try_from_sql_row)? + .collect::, _>>()?; + Ok(txs) + }) + .await } From a0369089c2250f728fd5530c2dd738a6c5509b1c Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 16 Aug 2024 12:31:39 +0200 Subject: [PATCH 083/128] working PoC for zcoin tx (history) streaming finally :) only for native, wasm to come --- .../utxo/utxo_builder/utxo_coin_builder.rs | 1 + mm2src/coins/z_coin/z_coin_native_tests.rs | 126 +++++++++++++++--- mm2src/coins/z_coin/z_tx_history.rs | 10 +- 3 files changed, 119 insertions(+), 18 deletions(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 7a112ffb41..106f78374e 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -775,6 +775,7 @@ pub trait UtxoCoinBuilderCommonOps { }; let secs_since_date = current_time_sec - date_s; + // FIXME: This panics subtract with overflow. let days_since_date = (secs_since_date / DAY_IN_SECONDS) - 1; let blocks_to_sync = (days_since_date * blocks_per_day) + blocks_per_day; diff --git a/mm2src/coins/z_coin/z_coin_native_tests.rs b/mm2src/coins/z_coin/z_coin_native_tests.rs index 9e4358727d..b97bb1d976 100644 --- a/mm2src/coins/z_coin/z_coin_native_tests.rs +++ b/mm2src/coins/z_coin/z_coin_native_tests.rs @@ -1,24 +1,70 @@ use bitcrypto::dhash160; +use common::custom_futures::timeout::FutureTimerExt; use common::{block_on, now_sec}; use mm2_core::mm_ctx::MmCtxBuilder; -use mm2_test_helpers::for_tests::zombie_conf; +use mm2_test_helpers::for_tests::{zombie_conf, ZOMBIE_TICKER}; use std::path::PathBuf; use std::time::Duration; use zcash_client_backend::encoding::decode_extended_spending_key; +use super::tx_history_events::ZCoinTxHistoryEventStreamer; use super::{z_coin_from_conf_and_params_with_z_key, z_mainnet_constants, Future, PrivKeyBuildPolicy, RefundPaymentArgs, SendPaymentArgs, SpendPaymentArgs, SwapOps, ValidateFeeArgs, ValidatePaymentError, ZTransaction}; +use crate::utxo::rpc_clients::ElectrumRpcRequest; +use crate::z_coin::z_coin_from_conf_and_params; use crate::z_coin::{z_htlc::z_send_dex_fee, ZcoinActivationParams, ZcoinRpcMode}; -use crate::DexFee; -use crate::{CoinProtocol, SwapTxTypeWithSecretHash}; +use crate::{CoinProtocol, MarketCoinOps, SwapTxTypeWithSecretHash}; +use crate::{DexFee, MmCoin}; use mm2_number::MmNumber; +fn native_zcoin_activation_params() -> ZcoinActivationParams { + ZcoinActivationParams { + mode: ZcoinRpcMode::Native, + required_confirmations: None, + requires_notarization: None, + zcash_params_path: None, + scan_blocks_per_iteration: 1000, + scan_interval_ms: 0, + account: 0, + } +} + +fn light_zcoin_activation_params() -> ZcoinActivationParams { + ZcoinActivationParams { + mode: ZcoinRpcMode::Light { + electrum_servers: ["zombie.dragonhound.info:10033", "zombie.dragonhound.info:10133"] + .iter() + .map(|s| ElectrumRpcRequest { + url: s.to_string(), + protocol: Default::default(), + disable_cert_verification: Default::default(), + }) + .collect(), + light_wallet_d_servers: [ + "https://zombie.dragonhound.info:443", + "https://zombie.dragonhound.info:1443", + ] + .iter() + .map(|s| s.to_string()) + .collect(), + sync_params: Some(crate::z_coin::SyncStartPoint::Date(now_sec() - 24 * 60 * 60)), + skip_sync_params: None, + }, + required_confirmations: None, + requires_notarization: None, + zcash_params_path: None, + scan_blocks_per_iteration: 1000, + scan_interval_ms: 0, + account: 0, + } +} + #[test] fn zombie_coin_send_and_refund_maker_payment() { let ctx = MmCtxBuilder::default().into_mm_arc(); let mut conf = zombie_conf(); - let params = default_zcoin_activation_params(); + let params = native_zcoin_activation_params(); let pk_data = [1; 32]; let db_dir = PathBuf::from("./for_tests"); let z_key = decode_extended_spending_key(z_mainnet_constants::HRP_SAPLING_EXTENDED_SPENDING_KEY, "secret-extended-key-main1q0k2ga2cqqqqpq8m8j6yl0say83cagrqp53zqz54w38ezs8ly9ly5ptamqwfpq85u87w0df4k8t2lwyde3n9v0gcr69nu4ryv60t0kfcsvkr8h83skwqex2nf0vr32794fmzk89cpmjptzc22lgu5wfhhp8lgf3f5vn2l3sge0udvxnm95k6dtxj2jwlfyccnum7nz297ecyhmd5ph526pxndww0rqq0qly84l635mec0x4yedf95hzn6kcgq8yxts26k98j9g32kjc8y83fe").unwrap().unwrap(); @@ -77,7 +123,7 @@ fn zombie_coin_send_and_refund_maker_payment() { fn zombie_coin_send_and_spend_maker_payment() { let ctx = MmCtxBuilder::default().into_mm_arc(); let mut conf = zombie_conf(); - let params = default_zcoin_activation_params(); + let params = native_zcoin_activation_params(); let pk_data = [1; 32]; let db_dir = PathBuf::from("./for_tests"); let z_key = decode_extended_spending_key(z_mainnet_constants::HRP_SAPLING_EXTENDED_SPENDING_KEY, "secret-extended-key-main1q0k2ga2cqqqqpq8m8j6yl0say83cagrqp53zqz54w38ezs8ly9ly5ptamqwfpq85u87w0df4k8t2lwyde3n9v0gcr69nu4ryv60t0kfcsvkr8h83skwqex2nf0vr32794fmzk89cpmjptzc22lgu5wfhhp8lgf3f5vn2l3sge0udvxnm95k6dtxj2jwlfyccnum7nz297ecyhmd5ph526pxndww0rqq0qly84l635mec0x4yedf95hzn6kcgq8yxts26k98j9g32kjc8y83fe").unwrap().unwrap(); @@ -139,7 +185,7 @@ fn zombie_coin_send_and_spend_maker_payment() { fn zombie_coin_send_dex_fee() { let ctx = MmCtxBuilder::default().into_mm_arc(); let mut conf = zombie_conf(); - let params = default_zcoin_activation_params(); + let params = native_zcoin_activation_params(); let priv_key = PrivKeyBuildPolicy::IguanaPrivKey([1; 32].into()); let db_dir = PathBuf::from("./for_tests"); let z_key = decode_extended_spending_key(z_mainnet_constants::HRP_SAPLING_EXTENDED_SPENDING_KEY, "secret-extended-key-main1q0k2ga2cqqqqpq8m8j6yl0say83cagrqp53zqz54w38ezs8ly9ly5ptamqwfpq85u87w0df4k8t2lwyde3n9v0gcr69nu4ryv60t0kfcsvkr8h83skwqex2nf0vr32794fmzk89cpmjptzc22lgu5wfhhp8lgf3f5vn2l3sge0udvxnm95k6dtxj2jwlfyccnum7nz297ecyhmd5ph526pxndww0rqq0qly84l635mec0x4yedf95hzn6kcgq8yxts26k98j9g32kjc8y83fe").unwrap().unwrap(); @@ -168,7 +214,7 @@ fn zombie_coin_send_dex_fee() { fn prepare_zombie_sapling_cache() { let ctx = MmCtxBuilder::default().into_mm_arc(); let mut conf = zombie_conf(); - let params = default_zcoin_activation_params(); + let params = native_zcoin_activation_params(); let priv_key = PrivKeyBuildPolicy::IguanaPrivKey([1; 32].into()); let db_dir = PathBuf::from("./for_tests"); let z_key = decode_extended_spending_key(z_mainnet_constants::HRP_SAPLING_EXTENDED_SPENDING_KEY, "secret-extended-key-main1q0k2ga2cqqqqpq8m8j6yl0say83cagrqp53zqz54w38ezs8ly9ly5ptamqwfpq85u87w0df4k8t2lwyde3n9v0gcr69nu4ryv60t0kfcsvkr8h83skwqex2nf0vr32794fmzk89cpmjptzc22lgu5wfhhp8lgf3f5vn2l3sge0udvxnm95k6dtxj2jwlfyccnum7nz297ecyhmd5ph526pxndww0rqq0qly84l635mec0x4yedf95hzn6kcgq8yxts26k98j9g32kjc8y83fe").unwrap().unwrap(); @@ -198,7 +244,7 @@ fn prepare_zombie_sapling_cache() { fn zombie_coin_validate_dex_fee() { let ctx = MmCtxBuilder::default().into_mm_arc(); let mut conf = zombie_conf(); - let params = default_zcoin_activation_params(); + let params = native_zcoin_activation_params(); let priv_key = PrivKeyBuildPolicy::IguanaPrivKey([1; 32].into()); let db_dir = PathBuf::from("./for_tests"); let z_key = decode_extended_spending_key(z_mainnet_constants::HRP_SAPLING_EXTENDED_SPENDING_KEY, "secret-extended-key-main1q0k2ga2cqqqqpq8m8j6yl0say83cagrqp53zqz54w38ezs8ly9ly5ptamqwfpq85u87w0df4k8t2lwyde3n9v0gcr69nu4ryv60t0kfcsvkr8h83skwqex2nf0vr32794fmzk89cpmjptzc22lgu5wfhhp8lgf3f5vn2l3sge0udvxnm95k6dtxj2jwlfyccnum7nz297ecyhmd5ph526pxndww0rqq0qly84l635mec0x4yedf95hzn6kcgq8yxts26k98j9g32kjc8y83fe").unwrap().unwrap(); @@ -282,14 +328,60 @@ fn zombie_coin_validate_dex_fee() { coin.validate_fee(validate_fee_args).wait().unwrap(); } -fn default_zcoin_activation_params() -> ZcoinActivationParams { - ZcoinActivationParams { - mode: ZcoinRpcMode::Native, - required_confirmations: None, - requires_notarization: None, - zcash_params_path: None, - scan_blocks_per_iteration: 0, - scan_interval_ms: 0, - account: 0, +#[test] +fn test_zombie_tx_streaming() { + let ctx = MmCtxBuilder::default().into_mm_arc(); + let conf = zombie_conf(); + let params = light_zcoin_activation_params(); + // Address: RQX5MnqnxEk6P33LSEAxC2vqA7DfSdWVyH + // Or: zs1n2azlwcj9pvl2eh36qvzgeukt2cpzmw44hya8wyu52j663d0dfs4d5hjx6tr04trz34jxyy433j + let priv_key_policy = + PrivKeyBuildPolicy::IguanaPrivKey("6d862798ef956fb60fb17bcc417dd6d44bfff066a4a49301cd2528e41a4a3e45".into()); + let protocol_info = match serde_json::from_value::(conf["protocol"].clone()).unwrap() { + CoinProtocol::ZHTLC(protocol_info) => protocol_info, + other_protocol => panic!("Failed to get protocol from config: {:?}", other_protocol), + }; + + let coin = block_on(z_coin_from_conf_and_params( + &ctx, + ZOMBIE_TICKER, + &conf, + ¶ms, + protocol_info, + priv_key_policy, + )) + .unwrap(); + + // Wait till we are synced with the sapling state. + while !block_on(coin.is_sapling_state_synced()) { + std::thread::sleep(Duration::from_secs(1)); } + + // Add a new client to use it for listening to tx history events. + let client_id = 1; + let mut event_receiver = ctx.event_stream_manager.new_client(client_id).unwrap(); + // Add the streamer that will stream the tx history events. + let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); + // Subscribe the client to the streamer. + block_on(ctx.event_stream_manager.add(client_id, streamer, coin.spawner())).unwrap(); + + // Send a tx to have it in the tx history. + let tx = block_on(z_send_dex_fee(&coin, "0.0001".parse().unwrap(), &[1; 16])).unwrap(); + + // Wait for the tx history event (should be streamed next block). + let event = block_on(Box::pin(event_receiver.recv()).timeout_secs(120.)) + .expect("timed out waiting for tx to showup") + .expect("tx history sender shutdown"); + + log!("{:?}", event.get()); + let (event_type, event_data) = event.get(); + // Make sure this is not an error event, + assert!(!event_type.starts_with("ERROR_")); + // from the expected streamer, + assert_eq!( + event_type, + ZCoinTxHistoryEventStreamer::derive_streamer_id(coin.ticker()) + ); + // and has the expected data. + assert_eq!(event_data["tx_hash"].as_str().unwrap(), tx.txid().to_string()); } diff --git a/mm2src/coins/z_coin/z_tx_history.rs b/mm2src/coins/z_coin/z_tx_history.rs index 2fa6077ea3..2125a23c7d 100644 --- a/mm2src/coins/z_coin/z_tx_history.rs +++ b/mm2src/coins/z_coin/z_tx_history.rs @@ -340,7 +340,15 @@ pub(crate) async fn fetch_txs_from_db( .field("blocks.time") .field("COALESCE(rn.received_amount, 0)") .field("COALESCE(sn.sent_amount, 0)") - .and_where_in("tx_hash", &tx_hashes.into_iter().collect::>()) + .and_where_in_quoted( + // Make sure the tx hash from the DB is lowercase, + "lower(hex(tx_hash))", + &tx_hashes + .iter() + // as well as the tx hashes we are looking for. + .map(|tx_hash| hex::encode(tx_hash.0).to_lowercase()) + .collect::>(), + ) .left() .join("(SELECT tx, SUM(value) as received_amount FROM received_notes GROUP BY tx) as rn") .on("txes.id_tx = rn.tx") From 873c20ec5bb3f1172c8ef9797752eb6d6b778ece Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 16 Aug 2024 12:40:57 +0200 Subject: [PATCH 084/128] zombie: fix a couple of pitfalls outdated urls, a runtime panic, non-zero scan blocks per iteration --- .gitignore | 3 +- .../utxo/utxo_builder/utxo_coin_builder.rs | 3 +- mm2src/coins/z_coin.rs | 24 ++++++++++++---- mm2src/coins/z_coin/z_coin_native_tests.rs | 12 ++------ mm2src/coins/z_coin/z_rpc.rs | 4 +-- mm2src/mm2_test_helpers/src/for_tests.rs | 28 +++++++++++-------- 6 files changed, 43 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index e7735b0e63..37c4968be4 100755 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,5 @@ MM2.json [._]sw[a-p] # mergetool -*.orig \ No newline at end of file +*.orig +hidden \ No newline at end of file diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 106f78374e..577f51855c 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -775,8 +775,7 @@ pub trait UtxoCoinBuilderCommonOps { }; let secs_since_date = current_time_sec - date_s; - // FIXME: This panics subtract with overflow. - let days_since_date = (secs_since_date / DAY_IN_SECONDS) - 1; + let days_since_date = (secs_since_date / DAY_IN_SECONDS).min(1) - 1; let blocks_to_sync = (days_since_date * blocks_per_day) + blocks_per_day; if current_block_height < blocks_to_sync { diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a9fd085846..d443fad4ec 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -46,9 +46,8 @@ use async_trait::async_trait; use bitcrypto::dhash256; use chain::constants::SEQUENCE_FINAL; use chain::{Transaction as UtxoTx, TransactionOutput}; -use common::calc_total_pages; use common::executor::{AbortableSystem, AbortedError}; -use common::{log, one_thousand_u32}; +use common::{calc_total_pages, log}; use crypto::privkey::{key_pair_from_secret, secp_privkey_from_hash}; use crypto::HDPathToCoin; use crypto::{Bip32DerPathOps, GlobalHDAccountArc}; @@ -70,6 +69,7 @@ use serialization::CoinVariant; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::iter; +use std::num::NonZeroU32; use std::path::PathBuf; use std::sync::Arc; pub use z_coin_errors::*; @@ -748,19 +748,31 @@ pub enum ZcoinRpcMode { } #[derive(Clone, Deserialize)] +#[serde(default)] pub struct ZcoinActivationParams { pub mode: ZcoinRpcMode, pub required_confirmations: Option, pub requires_notarization: Option, pub zcash_params_path: Option, - #[serde(default = "one_thousand_u32")] - pub scan_blocks_per_iteration: u32, - #[serde(default)] + pub scan_blocks_per_iteration: NonZeroU32, pub scan_interval_ms: u64, - #[serde(default)] pub account: u32, } +impl Default for ZcoinActivationParams { + fn default() -> Self { + Self { + mode: ZcoinRpcMode::Native, + required_confirmations: None, + requires_notarization: None, + zcash_params_path: None, + scan_blocks_per_iteration: NonZeroU32::new(1000).expect("1000 is a valid value"), + scan_interval_ms: 0, + account: 0, + } + } +} + pub async fn z_coin_from_conf_and_params( ctx: &MmArc, ticker: &str, diff --git a/mm2src/coins/z_coin/z_coin_native_tests.rs b/mm2src/coins/z_coin/z_coin_native_tests.rs index b97bb1d976..0aa35f6980 100644 --- a/mm2src/coins/z_coin/z_coin_native_tests.rs +++ b/mm2src/coins/z_coin/z_coin_native_tests.rs @@ -2,7 +2,7 @@ use bitcrypto::dhash160; use common::custom_futures::timeout::FutureTimerExt; use common::{block_on, now_sec}; use mm2_core::mm_ctx::MmCtxBuilder; -use mm2_test_helpers::for_tests::{zombie_conf, ZOMBIE_TICKER}; +use mm2_test_helpers::for_tests::{zombie_conf, ZOMBIE_ELECTRUMS, ZOMBIE_LIGHTWALLETD_URLS, ZOMBIE_TICKER}; use std::path::PathBuf; use std::time::Duration; use zcash_client_backend::encoding::decode_extended_spending_key; @@ -33,7 +33,7 @@ fn native_zcoin_activation_params() -> ZcoinActivationParams { fn light_zcoin_activation_params() -> ZcoinActivationParams { ZcoinActivationParams { mode: ZcoinRpcMode::Light { - electrum_servers: ["zombie.dragonhound.info:10033", "zombie.dragonhound.info:10133"] + electrum_servers: ZOMBIE_ELECTRUMS .iter() .map(|s| ElectrumRpcRequest { url: s.to_string(), @@ -41,13 +41,7 @@ fn light_zcoin_activation_params() -> ZcoinActivationParams { disable_cert_verification: Default::default(), }) .collect(), - light_wallet_d_servers: [ - "https://zombie.dragonhound.info:443", - "https://zombie.dragonhound.info:1443", - ] - .iter() - .map(|s| s.to_string()) - .collect(), + light_wallet_d_servers: ZOMBIE_LIGHTWALLETD_URLS.iter().map(|s| s.to_string()).collect(), sync_params: Some(crate::z_coin::SyncStartPoint::Date(now_sec() - 24 * 60 * 60)), skip_sync_params: None, }, diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 61f3d6a291..7bfd299bdc 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -567,7 +567,7 @@ pub(super) async fn init_light_client<'a>( main_sync_state_finished: false, on_tx_gen_watcher, watch_for_tx: None, - scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration, + scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration.into(), scan_interval_ms: builder.z_coin_params.scan_interval_ms, first_sync_block: first_sync_block.clone(), streaming_manager: builder.ctx.event_stream_manager.clone(), @@ -614,7 +614,7 @@ pub(super) async fn init_native_client<'a>( main_sync_state_finished: false, on_tx_gen_watcher, watch_for_tx: None, - scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration, + scan_blocks_per_iteration: builder.z_coin_params.scan_blocks_per_iteration.into(), scan_interval_ms: builder.z_coin_params.scan_interval_ms, first_sync_block: first_sync_block.clone(), streaming_manager: builder.ctx.event_stream_manager.clone(), diff --git a/mm2src/mm2_test_helpers/src/for_tests.rs b/mm2src/mm2_test_helpers/src/for_tests.rs index d207462f57..dd6b612695 100644 --- a/mm2src/mm2_test_helpers/src/for_tests.rs +++ b/mm2src/mm2_test_helpers/src/for_tests.rs @@ -194,21 +194,21 @@ pub const MARTY_ELECTRUM_ADDRS: &[&str] = &[ "electrum3.cipig.net:10021", ]; pub const ZOMBIE_TICKER: &str = "ZOMBIE"; +#[cfg(not(target_arch = "wasm32"))] +pub const ZOMBIE_ELECTRUMS: &[&str] = &["zombie.dragonhound.info:10033", "zombie.dragonhound.info:10133"]; +#[cfg(target_arch = "wasm32")] +pub const ZOMBIE_ELECTRUMS: &[&str] = &["zombie.dragonhound.info:30058", "zombie.dragonhound.info:30059"]; +pub const ZOMBIE_LIGHTWALLETD_URLS: &[&str] = &[ + "https://zombie.dragonhound.info:443", + "https://zombie.dragonhound.info:1443", +]; pub const ARRR: &str = "ARRR"; -pub const ZOMBIE_ELECTRUMS: &[&str] = &[ +#[cfg(not(target_arch = "wasm32"))] +pub const PIRATE_ELECTRUMS: &[&str] = &[ "electrum1.cipig.net:10008", "electrum2.cipig.net:10008", "electrum3.cipig.net:10008", ]; -pub const ZOMBIE_LIGHTWALLETD_URLS: &[&str] = &[ - "https://lightd1.pirate.black:443", - "https://piratelightd1.cryptoforge.cc:443", - "https://piratelightd2.cryptoforge.cc:443", - "https://piratelightd3.cryptoforge.cc:443", - "https://piratelightd4.cryptoforge.cc:443", -]; -#[cfg(not(target_arch = "wasm32"))] -pub const PIRATE_ELECTRUMS: &[&str] = &["node1.chainkeeper.pro:10132"]; #[cfg(target_arch = "wasm32")] pub const PIRATE_ELECTRUMS: &[&str] = &[ "electrum3.cipig.net:30008", @@ -216,7 +216,13 @@ pub const PIRATE_ELECTRUMS: &[&str] = &[ "electrum2.cipig.net:30008", ]; #[cfg(not(target_arch = "wasm32"))] -pub const PIRATE_LIGHTWALLETD_URLS: &[&str] = &["http://node1.chainkeeper.pro:443"]; +pub const PIRATE_LIGHTWALLETD_URLS: &[&str] = &[ + "https://lightd1.pirate.black:443", + "https://piratelightd1.cryptoforge.cc:443", + "https://piratelightd2.cryptoforge.cc:443", + "https://piratelightd3.cryptoforge.cc:443", + "https://piratelightd4.cryptoforge.cc:443", +]; #[cfg(target_arch = "wasm32")] pub const PIRATE_LIGHTWALLETD_URLS: &[&str] = &["https://pirate.battlefield.earth:8581"]; pub const DEFAULT_RPC_PASSWORD: &str = "pass"; From 746277d3d08799da90e0e3234ca40c2679062606 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 26 Aug 2024 09:55:06 +0200 Subject: [PATCH 085/128] abstract the ugly .0 tuple access --- mm2src/mm2_event_stream/src/manager.rs | 30 +++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 26eb532622..ecef8925d3 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -8,7 +8,7 @@ use common::log; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use tokio::sync::mpsc; /// The errors that could originate from the streaming manager. @@ -101,6 +101,12 @@ struct StreamingManagerInner { pub struct StreamingManager(Arc>); impl StreamingManager { + /// Returns a read guard over the streaming manager. + fn read(&self) -> RwLockReadGuard { self.0.read() } + + /// Returns a write guard over the streaming manager. + fn write(&self) -> RwLockWriteGuard { self.0.write() } + /// Spawns and adds a new streamer `streamer` to the manager. pub async fn add( &self, @@ -114,7 +120,7 @@ impl StreamingManager { // Pre-checks before spawning the streamer. Inside another scope to drop the lock early. { - let mut this = self.0.write(); + let mut this = self.write(); match this.clients.get(&client_id) { // We don't know that client. We don't have a connection to it. None => return Err(StreamingManagerError::UnknownClient), @@ -147,7 +153,7 @@ impl StreamingManager { // Note that we didn't hold the lock while spawning the streamer (potentially a long operation). // This means we can't assume either that the client still exists at this point or // that the streamer still doesn't exist. - let mut this = self.0.write(); + let mut this = self.write(); if let Some(client_info) = this.clients.get_mut(&client_id) { client_info.add_streamer(streamer_id.clone()); this.streamers @@ -164,7 +170,7 @@ impl StreamingManager { /// Sends data to a streamer with `streamer_id`. pub fn send(&self, streamer_id: &str, data: T) -> Result<(), StreamingManagerError> { - let this = self.0.read(); + let this = self.read(); let streamer_info = this .streamers .get(streamer_id) @@ -185,7 +191,7 @@ impl StreamingManager { streamer_id: &str, data_fn: impl FnOnce() -> T, ) -> Result<(), StreamingManagerError> { - let this = self.0.read(); + let this = self.read(); let streamer_info = this .streamers .get(streamer_id) @@ -198,7 +204,7 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { - let mut this = self.0.write(); + let mut this = self.write(); if let Some(client_info) = this.clients.get_mut(&client_id) { client_info.remove_streamer(streamer_id); @@ -224,7 +230,7 @@ impl StreamingManager { /// of any streamer (i.e. bypassing any streamer). pub fn broadcast(&self, event: Event) { let event = Arc::new(event); - let this = self.0.read(); + let this = self.read(); if let Some(client_ids) = this.streamers.get(event.origin()).map(|info| &info.clients) { client_ids.iter().for_each(|client_id| { this.clients.get(client_id).map(|info| info.send_event(event.clone())); @@ -235,14 +241,14 @@ impl StreamingManager { /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. pub fn broadcast_all(&self, event: Event) { let event = Arc::new(event); - self.0.read().clients.values().for_each(|info| { + self.read().clients.values().for_each(|info| { info.send_event(event.clone()); }); } /// Creates a new client and returns the event receiver for this client. pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { - let mut this = self.0.write(); + let mut this = self.write(); if this.clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists); } @@ -256,7 +262,7 @@ impl StreamingManager { /// Removes a client from the manager. pub fn remove_client(&self, client_id: u64) -> Result<(), StreamingManagerError> { - let mut this = self.0.write(); + let mut this = self.write(); // Remove the client from our known-clients map. let client_info = this .clients @@ -285,7 +291,7 @@ impl StreamingManager { /// the streamer might die by itself (e.g. the spawner it was spawned with aborted). /// In this case, we need to remove the streamer and de-list it from all clients. fn remove_streamer_if_down(&self, streamer_id: &str) { - let mut this = self.0.write(); + let mut this = self.write(); if let Some(streamer_info) = this.streamers.get_mut(streamer_id) { if streamer_info.is_down() { // Remove the streamer from our registry. @@ -483,7 +489,7 @@ mod tests { manager.remove_streamer_if_down(&streamer_id); // The streamer should be removed. - assert!(manager.0.read().streamers.get(&streamer_id).is_none()); + assert!(manager.read().streamers.get(&streamer_id).is_none()); // And the client is no more listening to it. assert!(!manager .0 From 1d0a75bd003c1d5b1cc4075900a9a50c091f5b4d Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 28 Aug 2024 13:15:57 +0200 Subject: [PATCH 086/128] fix: actually transfer something for order status events --- mm2src/mm2_main/src/lp_ordermatch.rs | 4 ++-- mm2src/mm2_main/src/lp_ordermatch/order_events.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index cc37e15330..7080736e7e 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -3893,7 +3893,7 @@ pub async fn sell(ctx: MmArc, req: Json) -> Result>, String> { /// Created when maker order is matched with taker request #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -struct MakerMatch { +pub struct MakerMatch { request: TakerRequest, reserved: MakerReserved, connect: Option, @@ -3903,7 +3903,7 @@ struct MakerMatch { /// Created upon taker request broadcast #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -struct TakerMatch { +pub struct TakerMatch { reserved: MakerReserved, connect: TakerConnect, connected: Option, diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 733d7bdba1..0ba4dc3905 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -15,7 +15,7 @@ impl OrderStatusStreamer { #[derive(Serialize)] #[serde(tag = "order_type", content = "order_data")] -pub(super) enum OrderStatusEvent { +pub enum OrderStatusEvent { MakerMatch(MakerMatch), TakerMatch(TakerMatch), MakerConnected(MakerMatch), @@ -24,7 +24,7 @@ pub(super) enum OrderStatusEvent { #[async_trait] impl EventStreamer for OrderStatusStreamer { - type DataInType = (); + type DataInType = OrderStatusEvent; fn streamer_id(&self) -> String { Self::derive_streamer_id().to_string() } @@ -38,8 +38,8 @@ impl EventStreamer for OrderStatusStreamer { .send(Ok(())) .expect("Receiver is dropped, which should never happen."); - while let Some(swap_data) = data_rx.next().await { - let event_data = serde_json::to_value(swap_data).expect("Serialization shouldn't fail."); + while let Some(order_data) = data_rx.next().await { + let event_data = serde_json::to_value(order_data).expect("Serialization shouldn't fail."); let event = Event::new(self.streamer_id(), event_data); broadcaster.broadcast(event); } From 329f152b5523935836f05c9e6bdca38c72acdb99 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 30 Aug 2024 16:02:09 +0300 Subject: [PATCH 087/128] suggestion from omar: don't nest --- mm2src/mm2_event_stream/src/manager.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 048452f3bc..8fd51435fc 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -205,22 +205,22 @@ impl StreamingManager { /// Stops streaming from the streamer with `streamer_id` to the client with `client_id`. pub fn stop(&self, client_id: u64, streamer_id: &str) -> Result<(), StreamingManagerError> { let mut this = self.write(); - if let Some(client_info) = this.clients.get_mut(&client_id) { - client_info.remove_streamer(streamer_id); + let client_info = this + .clients + .get_mut(&client_id) + .ok_or(StreamingManagerError::UnknownClient)?; + client_info.remove_streamer(streamer_id); - this.streamers - .get_mut(streamer_id) - .ok_or(StreamingManagerError::StreamerNotFound)? - .remove_client(&client_id); + this.streamers + .get_mut(streamer_id) + .ok_or(StreamingManagerError::StreamerNotFound)? + .remove_client(&client_id); - // If there are no more listening clients, terminate the streamer. - if this.streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { - this.streamers.remove(streamer_id); - } - Ok(()) - } else { - Err(StreamingManagerError::UnknownClient) + // If there are no more listening clients, terminate the streamer. + if this.streamers.get(streamer_id).map(|info| info.clients.len()) == Some(0) { + this.streamers.remove(streamer_id); } + Ok(()) } /// Broadcasts some event to clients listening to it. From 349c5e9824afd8051ca353c5698acf21193d6d56 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 30 Aug 2024 17:56:21 +0300 Subject: [PATCH 088/128] fix wasm compilation impl zcoin activation params with lightmode as default since native won't work on wasm --- mm2src/coins/z_coin.rs | 7 ++++++- mm2src/mm2_event_stream/src/manager.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index d443fad4ec..540de538df 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -762,7 +762,12 @@ pub struct ZcoinActivationParams { impl Default for ZcoinActivationParams { fn default() -> Self { Self { - mode: ZcoinRpcMode::Native, + mode: ZcoinRpcMode::Light { + electrum_servers: Vec::new(), + light_wallet_d_servers: Vec::new(), + sync_params: None, + skip_sync_params: None, + }, required_confirmations: None, requires_notarization: None, zcash_params_path: None, diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 8fd51435fc..6bf0db8d65 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -388,7 +388,7 @@ mod tests { // We should be hooked now. try to receive some events from the streamer. for _ in 0..3 { - // The streamer should send an event every 0.1s. Wait for 0.2s for safety. + // The streamer should send an event every 0.1s. Wait for 0.15s for safety. Timer::sleep(0.15).await; let event = client1.try_recv().unwrap(); assert_eq!(event.origin(), streamer_id); From 9f63e6fd291857c30db558b8f589d253525c65c6 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 5 Sep 2024 13:30:59 +0300 Subject: [PATCH 089/128] fix: test_hd_utxo_tx_history the ctx wasn't hooked up, unwrap panics when accessing the streaming manager in the background thread --- mm2src/coins/utxo/utxo_common_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index 9f3b1d7a1f..bffe3d6a7b 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -270,6 +270,7 @@ pub(super) async fn test_hd_utxo_tx_history_impl(rpc_client: ElectrumClient) { hd_accounts.insert(0, hd_account_for_test); let mut fields = utxo_coin_fields_for_test(rpc_client.into(), None, false); + fields.ctx = ctx.weak(); fields.conf.ticker = "DOC".to_string(); fields.derivation_method = DerivationMethod::HDWallet(UtxoHDWallet { inner: HDWallet { From bcd4e1c9fefc8117213f51e8720c9586b9bca00f Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 5 Sep 2024 13:49:09 +0300 Subject: [PATCH 090/128] fix: activate_z_coin_light_with_changing_height i confused min with max when fixing this potential panic. max is the one we want here, so that when we have 0.max(1) we get 1 and not 0 --- mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 577f51855c..007a44068b 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -775,7 +775,7 @@ pub trait UtxoCoinBuilderCommonOps { }; let secs_since_date = current_time_sec - date_s; - let days_since_date = (secs_since_date / DAY_IN_SECONDS).min(1) - 1; + let days_since_date = (secs_since_date / DAY_IN_SECONDS).max(1) - 1; let blocks_to_sync = (days_since_date * blocks_per_day) + blocks_per_day; if current_block_height < blocks_to_sync { From 3f212c4da5430b2b587bf920ca8b9b001e3250a5 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 11 Sep 2024 09:56:31 +0300 Subject: [PATCH 091/128] add `broadcast_to` to send an event to a single client without any streamer involved --- mm2src/mm2_event_stream/src/manager.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 6bf0db8d65..38552836f4 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -240,6 +240,18 @@ impl StreamingManager { }; } + /// Broadcasts (actually just *sends* in this case) some event to a specific client. + /// + /// Could be used in case we have a single known client and don't want to spawn up a streamer just for that. + pub fn broadcast_to(&self, event: Event, client_id: u64) -> Result<(), StreamingManagerError> { + let event = Arc::new(event); + self.read() + .clients + .get(&client_id) + .map(|info| info.send_event(event)) + .ok_or(StreamingManagerError::UnknownClient) + } + /// Forcefully broadcasts an event to all known clients even if they are not listening for such an event. pub fn broadcast_all(&self, event: Event) { let event = Arc::new(event); From 2540e837b32c5d27f3c84ef9e677b3428ecfa082 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 11 Sep 2024 13:14:28 +0300 Subject: [PATCH 092/128] event streaming for task managed API end points this is a generic impl (and i think is better), and doesn't require a subscription (because there are no streamers running for it). the flow goes as follows: - the client inits a task managed request - the client (all clients actually) receive updates once they happen (either action required, request resolution (ok, err), etc...) later we should make it so that the client ID is provided in the init request, but optionally, and if the client provides it, only them will receive the updates, otherwise we can choose to broadcast_all or not broadcast at all. Note that spawning a streamer for each single task and hooking it up to a single client is tedious, for this we can instead broadcast the udpate rightaway from the rpc task manager knowing the client ID using the new `broadcast_to`. --- Cargo.lock | 2 + mm2src/coins/lp_coins.rs | 10 ++-- mm2src/coins_activation/src/context.rs | 14 +++--- mm2src/coins_activation/src/init_token.rs | 6 +-- mm2src/coins_activation/src/l2/init_l2.rs | 4 +- .../src/platform_coin_with_tokens.rs | 4 +- mm2src/mm2_main/src/lp_init/init_context.rs | 4 +- mm2src/rpc_task/Cargo.toml | 2 + mm2src/rpc_task/src/manager.rs | 48 +++++++++++++------ mm2src/rpc_task/src/task.rs | 4 +- 10 files changed, 61 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2dc3873d7d..3144d79a8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6365,10 +6365,12 @@ dependencies = [ "derive_more", "futures 0.3.28", "mm2_err_handle", + "mm2_event_stream", "ser_error", "ser_error_derive", "serde", "serde_derive", + "serde_json", ] [[package]] diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index a6a5a9ee78..186e599ad1 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -3800,11 +3800,11 @@ impl CoinsContext { platform_coin_tokens: PaMutex::new(HashMap::new()), coins: AsyncMutex::new(HashMap::new()), balance_update_handlers: AsyncMutex::new(vec![]), - account_balance_task_manager: AccountBalanceTaskManager::new_shared(), - create_account_manager: CreateAccountTaskManager::new_shared(), - get_new_address_manager: GetNewAddressTaskManager::new_shared(), - scan_addresses_manager: ScanAddressesTaskManager::new_shared(), - withdraw_task_manager: WithdrawTaskManager::new_shared(), + account_balance_task_manager: AccountBalanceTaskManager::new_shared(ctx.event_stream_manager.clone()), + create_account_manager: CreateAccountTaskManager::new_shared(ctx.event_stream_manager.clone()), + get_new_address_manager: GetNewAddressTaskManager::new_shared(ctx.event_stream_manager.clone()), + scan_addresses_manager: ScanAddressesTaskManager::new_shared(ctx.event_stream_manager.clone()), + withdraw_task_manager: WithdrawTaskManager::new_shared(ctx.event_stream_manager.clone()), #[cfg(target_arch = "wasm32")] tx_history_db: ConstructibleDb::new(ctx).into_shared(), #[cfg(target_arch = "wasm32")] diff --git a/mm2src/coins_activation/src/context.rs b/mm2src/coins_activation/src/context.rs index 5ae19eb60e..b26f24e3b6 100644 --- a/mm2src/coins_activation/src/context.rs +++ b/mm2src/coins_activation/src/context.rs @@ -28,14 +28,14 @@ impl CoinsActivationContext { from_ctx(&ctx.coins_activation_ctx, move || { Ok(CoinsActivationContext { #[cfg(feature = "enable-sia")] - init_sia_task_manager: RpcTaskManager::new_shared(), - init_utxo_standard_task_manager: RpcTaskManager::new_shared(), - init_qtum_task_manager: RpcTaskManager::new_shared(), - init_z_coin_task_manager: RpcTaskManager::new_shared(), - init_eth_task_manager: RpcTaskManager::new_shared(), - init_erc20_token_task_manager: RpcTaskManager::new_shared(), + init_sia_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_utxo_standard_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_qtum_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_z_coin_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_eth_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_erc20_token_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), #[cfg(not(target_arch = "wasm32"))] - init_lightning_task_manager: RpcTaskManager::new_shared(), + init_lightning_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), }) }) } diff --git a/mm2src/coins_activation/src/init_token.rs b/mm2src/coins_activation/src/init_token.rs index dbc03b1754..bdf54d58a9 100644 --- a/mm2src/coins_activation/src/init_token.rs +++ b/mm2src/coins_activation/src/init_token.rs @@ -45,7 +45,7 @@ pub struct InitTokenReq { pub trait InitTokenActivationOps: Into + TokenOf + Clone + Send + Sync + 'static { type ActivationRequest: Clone + Send + Sync; type ProtocolInfo: TokenProtocolParams + TryFromCoinProtocol + Clone + Send + Sync; - type ActivationResult: serde::Serialize + Clone + CurrentBlock + Send + Sync; + type ActivationResult: CurrentBlock + serde::Serialize + Clone + Send + Sync; type ActivationError: From + Into + NotEqual @@ -53,8 +53,8 @@ pub trait InitTokenActivationOps: Into + TokenOf + Clone + Send + Sy + Clone + Send + Sync; - type InProgressStatus: InitTokenInitialStatus + Clone + Send + Sync; - type AwaitingStatus: Clone + Send + Sync; + type InProgressStatus: InitTokenInitialStatus + serde::Serialize + Clone + Send + Sync; + type AwaitingStatus: serde::Serialize + Clone + Send + Sync; type UserAction: NotMmError + Send + Sync; /// Getter for the token initialization task manager. diff --git a/mm2src/coins_activation/src/l2/init_l2.rs b/mm2src/coins_activation/src/l2/init_l2.rs index 20e66ebbde..4af1038f41 100644 --- a/mm2src/coins_activation/src/l2/init_l2.rs +++ b/mm2src/coins_activation/src/l2/init_l2.rs @@ -39,8 +39,8 @@ pub trait InitL2ActivationOps: Into + Send + Sync + 'static { type CoinConf: Clone + Send + Sync; type ActivationResult: serde::Serialize + Clone + Send + Sync; type ActivationError: From + NotEqual + SerMmErrorType + Clone + Send + Sync; - type InProgressStatus: InitL2InitialStatus + Clone + Send + Sync; - type AwaitingStatus: Clone + Send + Sync; + type InProgressStatus: InitL2InitialStatus + serde::Serialize + Clone + Send + Sync; + type AwaitingStatus: serde::Serialize + Clone + Send + Sync; type UserAction: NotMmError + Send + Sync; fn rpc_task_manager(activation_ctx: &CoinsActivationContext) -> &InitL2TaskManagerShared; diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index 83566123db..98563b30fb 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -171,8 +171,8 @@ pub trait PlatformCoinWithTokensActivationOps: Into + Clone + Send + + Send + Sync; - type InProgressStatus: InitPlatformCoinWithTokensInitialStatus + Clone + Send + Sync; - type AwaitingStatus: Clone + Send + Sync; + type InProgressStatus: InitPlatformCoinWithTokensInitialStatus + serde::Serialize + Clone + Send + Sync; + type AwaitingStatus: serde::Serialize + Clone + Send + Sync; type UserAction: NotMmError + Send + Sync; /// Initializes the platform coin itself diff --git a/mm2src/mm2_main/src/lp_init/init_context.rs b/mm2src/mm2_main/src/lp_init/init_context.rs index 8b03751b69..a260b4ab67 100644 --- a/mm2src/mm2_main/src/lp_init/init_context.rs +++ b/mm2src/mm2_main/src/lp_init/init_context.rs @@ -16,9 +16,9 @@ impl MmInitContext { pub fn from_ctx(ctx: &MmArc) -> Result, String> { from_ctx(&ctx.mm_init_ctx, move || { Ok(MmInitContext { - init_hw_task_manager: RpcTaskManager::new_shared(), + init_hw_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), #[cfg(target_arch = "wasm32")] - init_metamask_manager: RpcTaskManager::new_shared(), + init_metamask_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), }) }) } diff --git a/mm2src/rpc_task/Cargo.toml b/mm2src/rpc_task/Cargo.toml index 4bf524f86a..c542159f25 100644 --- a/mm2src/rpc_task/Cargo.toml +++ b/mm2src/rpc_task/Cargo.toml @@ -10,9 +10,11 @@ doctest = false async-trait = "0.1" common = { path = "../common" } mm2_err_handle = { path = "../mm2_err_handle" } +mm2_event_stream = { path = "../mm2_event_stream" } derive_more = "0.99" futures = "0.3" ser_error = { path = "../derives/ser_error" } ser_error_derive = { path = "../derives/ser_error_derive" } serde = "1" serde_derive = "1" +serde_json = "1" diff --git a/mm2src/rpc_task/src/manager.rs b/mm2src/rpc_task/src/manager.rs index 950eac97f4..fae5cf6400 100644 --- a/mm2src/rpc_task/src/manager.rs +++ b/mm2src/rpc_task/src/manager.rs @@ -6,6 +6,7 @@ use common::log::{debug, info}; use futures::channel::oneshot; use futures::future::{select, Either}; use mm2_err_handle::prelude::*; +use mm2_event_stream::{Event, StreamingManager}; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::atomic::Ordering; @@ -29,11 +30,10 @@ static NEXT_RPC_TASK_ID: AtomicTaskId = AtomicTaskId::new(0); fn next_rpc_task_id() -> TaskId { NEXT_RPC_TASK_ID.fetch_add(1, Ordering::Relaxed) } pub struct RpcTaskManager { + /// A map of task IDs to their statuses and abort handlers. tasks: HashMap>, -} - -impl Default for RpcTaskManager { - fn default() -> Self { RpcTaskManager { tasks: HashMap::new() } } + /// A copy of the MM2's streaming manager to broadcast task status updates to interested parties. + streaming_manager: StreamingManager, } impl RpcTaskManager { @@ -43,12 +43,11 @@ impl RpcTaskManager { where F: SpawnFuture, { - let initial_task_status = task.initial_status(); let (task_id, task_abort_handler) = { let mut task_manager = this .lock() .map_to_mm(|e| RpcTaskError::Internal(format!("RpcTaskManager is not available: {}", e)))?; - task_manager.register_task(initial_task_status)? + task_manager.register_task(&task)? }; let task_handle = Arc::new(RpcTaskHandle { task_manager: RpcTaskManagerShared::downgrade(this), @@ -103,7 +102,16 @@ impl RpcTaskManager { Some(rpc_status) } - pub fn new_shared() -> RpcTaskManagerShared { Arc::new(Mutex::new(Self::default())) } + pub fn new(streaming_manager: StreamingManager) -> Self { + RpcTaskManager { + tasks: HashMap::new(), + streaming_manager, + } + } + + pub fn new_shared(streaming_manager: StreamingManager) -> RpcTaskManagerShared { + Arc::new(Mutex::new(Self::new(streaming_manager))) + } pub fn contains(&self, task_id: TaskId) -> bool { self.tasks.contains_key(&task_id) } @@ -138,17 +146,17 @@ impl RpcTaskManager { } } - pub(crate) fn register_task( - &mut self, - task_initial_in_progress_status: Task::InProgressStatus, - ) -> RpcTaskResult<(TaskId, TaskAbortHandler)> { + // FIXME: We can ask &Task for `client_id() -> Option` (most Tasks stores `req/request` and we can + // have the client provide these in their "task::" request so we know who to respond to). + // We can then store this ID in our map and use it to send updates to that client. Or send nothing if not provided. + pub(crate) fn register_task(&mut self, task: &Task) -> RpcTaskResult<(TaskId, TaskAbortHandler)> { let task_id = next_rpc_task_id(); let (abort_handle, abort_handler) = oneshot::channel(); match self.tasks.entry(task_id) { Entry::Occupied(_entry) => unexpected_task_status!(task_id, actual = InProgress, expected = Idle), Entry::Vacant(entry) => { entry.insert(TaskStatusExt::InProgress { - status: task_initial_in_progress_status, + status: task.initial_status(), abort_handle, }); Ok((task_id, abort_handler)) @@ -157,7 +165,7 @@ impl RpcTaskManager { } pub(crate) fn update_task_status(&mut self, task_id: TaskId, status: TaskStatus) -> RpcTaskResult<()> { - match status { + let update_result = match status { TaskStatus::Ok(result) => self.on_task_finished(task_id, Ok(result)), TaskStatus::Error(error) => self.on_task_finished(task_id, Err(error)), TaskStatus::InProgress(in_progress) => self.update_in_progress_status(task_id, in_progress), @@ -165,7 +173,19 @@ impl RpcTaskManager { awaiting_status, user_action_tx, } => self.set_task_is_waiting_for_user_action(task_id, awaiting_status, user_action_tx), - } + }; + // If the status was updated successfully, we need to inform the client about the new status. + if update_result.is_ok() { + // Note that this should really always be Some, since we updated the status successfully. + if let Some(new_status) = self.task_status(task_id, false) { + let event = Event::new( + format!("TASK:{task_id}"), + serde_json::to_value(new_status).expect("Serialization shouldn't fail."), + ); + self.streaming_manager.broadcast_all(event); + }; + }; + update_result } pub(crate) fn on_task_cancelling_finished(&mut self, task_id: TaskId) -> RpcTaskResult<()> { diff --git a/mm2src/rpc_task/src/task.rs b/mm2src/rpc_task/src/task.rs index 6c38f75050..3ec372041a 100644 --- a/mm2src/rpc_task/src/task.rs +++ b/mm2src/rpc_task/src/task.rs @@ -6,8 +6,8 @@ use serde::Serialize; pub trait RpcTaskTypes { type Item: Serialize + Clone + Send + Sync + 'static; type Error: SerMmErrorType + Clone + Send + Sync + 'static; - type InProgressStatus: Clone + Send + Sync + 'static; - type AwaitingStatus: Clone + Send + Sync + 'static; + type InProgressStatus: Serialize + Clone + Send + Sync + 'static; + type AwaitingStatus: Serialize + Clone + Send + Sync + 'static; type UserAction: NotMmError + Send + Sync + 'static; } From 12c5b41006d0a1b35ec3e4a22d59462cca229a91 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 11 Sep 2024 14:15:10 +0300 Subject: [PATCH 093/128] stream task status update only to the requesting client `task::` API init messages now have an optional `client_id` which can be used to stream out task status upadates and results to the client via event streaming. the client must have already done the inital SSE request in case of native and they must use the same client_id they used in that SSE init request. --- mm2src/coins/lp_coins.rs | 1 + mm2src/coins/rpc_command/get_new_address.rs | 3 + .../coins/rpc_command/init_account_balance.rs | 3 + .../coins/rpc_command/init_create_account.rs | 3 + .../init_scan_for_new_addresses.rs | 3 + mm2src/coins/rpc_command/init_withdraw.rs | 2 + mm2src/coins_activation/src/init_token.rs | 3 + mm2src/coins_activation/src/l2/init_l2.rs | 6 ++ .../src/platform_coin_with_tokens.rs | 3 + .../standalone_coin/init_standalone_coin.rs | 3 + mm2src/mm2_main/src/lp_init/init_hw.rs | 3 + mm2src/mm2_main/src/lp_init/init_metamask.rs | 8 +- mm2src/rpc_task/src/manager.rs | 74 +++++++++++++------ mm2src/rpc_task/src/task.rs | 5 ++ 14 files changed, 97 insertions(+), 23 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 186e599ad1..7a631d2253 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -2177,6 +2177,7 @@ pub struct WithdrawRequest { #[cfg(target_arch = "wasm32")] #[serde(default)] broadcast: bool, + client_id: Option, } #[derive(Debug, Deserialize)] diff --git a/mm2src/coins/rpc_command/get_new_address.rs b/mm2src/coins/rpc_command/get_new_address.rs index 9bba74cf75..aa3636f918 100644 --- a/mm2src/coins/rpc_command/get_new_address.rs +++ b/mm2src/coins/rpc_command/get_new_address.rs @@ -204,6 +204,7 @@ pub struct GetNewAddressRequest { coin: String, #[serde(flatten)] params: GetNewAddressParams, + client_id: Option, } #[derive(Clone, Deserialize)] @@ -291,6 +292,8 @@ impl RpcTaskTypes for InitGetNewAddressTask { impl RpcTask for InitGetNewAddressTask { fn initial_status(&self) -> Self::InProgressStatus { GetNewAddressInProgressStatus::Preparing } + fn client_id(&self) -> Option { self.req.client_id } + // Do nothing if the task has been cancelled. async fn cancel(self) {} diff --git a/mm2src/coins/rpc_command/init_account_balance.rs b/mm2src/coins/rpc_command/init_account_balance.rs index 94745f65e5..e7d7cea183 100644 --- a/mm2src/coins/rpc_command/init_account_balance.rs +++ b/mm2src/coins/rpc_command/init_account_balance.rs @@ -31,6 +31,7 @@ pub struct InitAccountBalanceRequest { coin: String, #[serde(flatten)] params: InitAccountBalanceParams, + client_id: Option, } #[derive(Clone, Deserialize)] @@ -65,6 +66,8 @@ impl RpcTaskTypes for InitAccountBalanceTask { impl RpcTask for InitAccountBalanceTask { fn initial_status(&self) -> Self::InProgressStatus { AccountBalanceInProgressStatus::RequestingAccountBalance } + fn client_id(&self) -> Option { self.req.client_id } + // Do nothing if the task has been cancelled. async fn cancel(self) {} diff --git a/mm2src/coins/rpc_command/init_create_account.rs b/mm2src/coins/rpc_command/init_create_account.rs index 012009d04b..695b3e5170 100644 --- a/mm2src/coins/rpc_command/init_create_account.rs +++ b/mm2src/coins/rpc_command/init_create_account.rs @@ -164,6 +164,7 @@ pub struct CreateNewAccountRequest { coin: String, #[serde(flatten)] params: CreateNewAccountParams, + client_id: Option, } #[derive(Clone, Deserialize)] @@ -238,6 +239,8 @@ impl RpcTaskTypes for InitCreateAccountTask { impl RpcTask for InitCreateAccountTask { fn initial_status(&self) -> Self::InProgressStatus { CreateAccountInProgressStatus::Preparing } + fn client_id(&self) -> Option { self.req.client_id } + async fn cancel(self) { if let Some(account_id) = self.task_state.create_account_id() { // We created the account already, so need to revert the changes. diff --git a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs index 4eabda46e9..ebbb4d4b86 100644 --- a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs +++ b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs @@ -43,6 +43,7 @@ pub struct ScanAddressesRequest { coin: String, #[serde(flatten)] params: ScanAddressesParams, + client_id: Option, } #[derive(Clone, Deserialize)] @@ -87,6 +88,8 @@ impl RpcTask for InitScanAddressesTask { #[inline] fn initial_status(&self) -> Self::InProgressStatus { ScanAddressesInProgressStatus::InProgress } + fn client_id(&self) -> Option { self.req.client_id } + // Do nothing if the task has been cancelled. async fn cancel(self) {} diff --git a/mm2src/coins/rpc_command/init_withdraw.rs b/mm2src/coins/rpc_command/init_withdraw.rs index 43b86cf19a..09728d4785 100644 --- a/mm2src/coins/rpc_command/init_withdraw.rs +++ b/mm2src/coins/rpc_command/init_withdraw.rs @@ -123,6 +123,8 @@ impl RpcTaskTypes for WithdrawTask { impl RpcTask for WithdrawTask { fn initial_status(&self) -> Self::InProgressStatus { WithdrawInProgressStatus::Preparing } + fn client_id(&self) -> Option { self.request.client_id } + // Do nothing if the task has been cancelled. async fn cancel(self) {} diff --git a/mm2src/coins_activation/src/init_token.rs b/mm2src/coins_activation/src/init_token.rs index bdf54d58a9..38e433285f 100644 --- a/mm2src/coins_activation/src/init_token.rs +++ b/mm2src/coins_activation/src/init_token.rs @@ -38,6 +38,7 @@ pub type CancelInitTokenError = CancelRpcTaskError; pub struct InitTokenReq { ticker: String, activation_params: T, + client_id: Option, } /// Trait for the initializing a token using the task manager. @@ -195,6 +196,8 @@ where ::initial_status() } + fn client_id(&self) -> Option { self.request.client_id } + /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(c_ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/coins_activation/src/l2/init_l2.rs b/mm2src/coins_activation/src/l2/init_l2.rs index 4af1038f41..ab59fc2a6c 100644 --- a/mm2src/coins_activation/src/l2/init_l2.rs +++ b/mm2src/coins_activation/src/l2/init_l2.rs @@ -24,6 +24,7 @@ pub type InitL2TaskHandleShared = RpcTaskHandleShared>; pub struct InitL2Req { ticker: String, activation_params: T, + client_id: Option, } pub trait L2ProtocolParams { @@ -98,9 +99,11 @@ where let coins_act_ctx = CoinsActivationContext::from_ctx(&ctx).map_to_mm(InitL2Error::Internal)?; let spawner = ctx.spawner(); + let client_id = req.client_id; let task = InitL2Task:: { ctx, ticker, + client_id, platform_coin, validated_params, protocol_conf, @@ -161,6 +164,7 @@ pub async fn cancel_init_l2( pub struct InitL2Task { ctx: MmArc, ticker: String, + client_id: Option, platform_coin: L2::PlatformCoin, validated_params: L2::ValidatedParams, protocol_conf: L2::ProtocolInfo, @@ -184,6 +188,8 @@ where ::initial_status() } + fn client_id(&self) -> Option { self.client_id } + /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index 98563b30fb..0c0c1f6106 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -223,6 +223,7 @@ pub struct EnablePlatformCoinWithTokensReq { ticker: String, #[serde(flatten)] request: T, + client_id: Option, } #[derive(Debug, Display, Serialize, SerializeErrorType, Clone)] @@ -509,6 +510,8 @@ where ::initial_status() } + fn client_id(&self) -> Option { self.request.client_id } + /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) {} diff --git a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs index 314f3066b4..3288832add 100644 --- a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs +++ b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs @@ -28,6 +28,7 @@ pub type InitStandaloneCoinTaskHandleShared = RpcTaskHandleShared { ticker: String, activation_params: T, + client_id: Option, } #[async_trait] @@ -183,6 +184,8 @@ where ::initial_status() } + fn client_id(&self) -> Option { self.request.client_id } + /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(c_ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/mm2_main/src/lp_init/init_hw.rs b/mm2src/mm2_main/src/lp_init/init_hw.rs index b9d0c67664..0315b17aa3 100644 --- a/mm2src/mm2_main/src/lp_init/init_hw.rs +++ b/mm2src/mm2_main/src/lp_init/init_hw.rs @@ -101,6 +101,7 @@ pub enum InitHwInProgressStatus { #[derive(Deserialize, Clone)] pub struct InitHwRequest { device_pubkey: Option, + client_id: Option, } #[derive(Clone, Serialize, Debug, Deserialize)] @@ -129,6 +130,8 @@ impl RpcTaskTypes for InitHwTask { impl RpcTask for InitHwTask { fn initial_status(&self) -> Self::InProgressStatus { InitHwInProgressStatus::Initializing } + fn client_id(&self) -> Option { self.req.client_id } + async fn cancel(self) { if let Ok(crypto_ctx) = CryptoCtx::from_ctx(&self.ctx) { crypto_ctx.reset_hw_ctx() diff --git a/mm2src/mm2_main/src/lp_init/init_metamask.rs b/mm2src/mm2_main/src/lp_init/init_metamask.rs index 630f6a1796..0fc5a0db36 100644 --- a/mm2src/mm2_main/src/lp_init/init_metamask.rs +++ b/mm2src/mm2_main/src/lp_init/init_metamask.rs @@ -11,9 +11,10 @@ use mm2_err_handle::common_errors::WithInternal; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; -use std::time::Duration; +use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, + RpcTaskTypes}; use std::sync::Arc; +use std::time::Duration; pub type InitMetamaskManagerShared = RpcTaskManagerShared; pub type InitMetamaskStatus = @@ -90,6 +91,7 @@ pub enum InitMetamaskInProgressStatus { #[derive(Deserialize)] pub struct InitMetamaskRequest { project: String, + client_id: Option, } #[derive(Clone, Serialize)] @@ -114,6 +116,8 @@ impl RpcTaskTypes for InitMetamaskTask { impl RpcTask for InitMetamaskTask { fn initial_status(&self) -> Self::InProgressStatus { InitMetamaskInProgressStatus::Initializing } + fn client_id(&self) -> Option { self.req.client_id } + async fn cancel(self) { if let Ok(crypto_ctx) = CryptoCtx::from_ctx(&self.ctx) { crypto_ctx.reset_metamask_ctx(); diff --git a/mm2src/rpc_task/src/manager.rs b/mm2src/rpc_task/src/manager.rs index fae5cf6400..12f1a4ed23 100644 --- a/mm2src/rpc_task/src/manager.rs +++ b/mm2src/rpc_task/src/manager.rs @@ -2,7 +2,7 @@ use crate::task::RpcTaskTypes; use crate::{AtomicTaskId, RpcTask, RpcTaskError, RpcTaskHandle, RpcTaskResult, RpcTaskStatus, RpcTaskStatusAlias, TaskAbortHandle, TaskAbortHandler, TaskId, TaskStatus, TaskStatusError, UserActionSender}; use common::executor::SpawnFuture; -use common::log::{debug, info}; +use common::log::{debug, info, warn}; use futures::channel::oneshot; use futures::future::{select, Either}; use mm2_err_handle::prelude::*; @@ -115,6 +115,13 @@ impl RpcTaskManager { pub fn contains(&self, task_id: TaskId) -> bool { self.tasks.contains_key(&task_id) } + fn get_client_id(&self, task_id: TaskId) -> Option { + self.tasks.get(&task_id).and_then(|task| match task { + TaskStatusExt::InProgress { client_id, .. } | TaskStatusExt::Awaiting { client_id, .. } => *client_id, + _ => None, + }) + } + /// Cancel task if it's in progress. pub fn cancel_task(&mut self, task_id: TaskId) -> RpcTaskResult<()> { let task = self.tasks.remove(&task_id); @@ -146,9 +153,6 @@ impl RpcTaskManager { } } - // FIXME: We can ask &Task for `client_id() -> Option` (most Tasks stores `req/request` and we can - // have the client provide these in their "task::" request so we know who to respond to). - // We can then store this ID in our map and use it to send updates to that client. Or send nothing if not provided. pub(crate) fn register_task(&mut self, task: &Task) -> RpcTaskResult<(TaskId, TaskAbortHandler)> { let task_id = next_rpc_task_id(); let (abort_handle, abort_handler) = oneshot::channel(); @@ -158,6 +162,7 @@ impl RpcTaskManager { entry.insert(TaskStatusExt::InProgress { status: task.initial_status(), abort_handle, + client_id: task.client_id(), }); Ok((task_id, abort_handler)) }, @@ -165,6 +170,8 @@ impl RpcTaskManager { } pub(crate) fn update_task_status(&mut self, task_id: TaskId, status: TaskStatus) -> RpcTaskResult<()> { + // Get the client ID before updating the task status because not all task status variants store the ID. + let client_id = self.get_client_id(task_id); let update_result = match status { TaskStatus::Ok(result) => self.on_task_finished(task_id, Ok(result)), TaskStatus::Error(error) => self.on_task_finished(task_id, Err(error)), @@ -176,14 +183,18 @@ impl RpcTaskManager { }; // If the status was updated successfully, we need to inform the client about the new status. if update_result.is_ok() { - // Note that this should really always be Some, since we updated the status successfully. - if let Some(new_status) = self.task_status(task_id, false) { - let event = Event::new( - format!("TASK:{task_id}"), - serde_json::to_value(new_status).expect("Serialization shouldn't fail."), - ); - self.streaming_manager.broadcast_all(event); - }; + if let Some(client_id) = client_id { + // Note that this should really always be `Some`, since we updated the status *successfully*. + if let Some(new_status) = self.task_status(task_id, false) { + let event = Event::new( + format!("TASK:{task_id}"), + serde_json::to_value(new_status).expect("Serialization shouldn't fail."), + ); + if let Err(e) = self.streaming_manager.broadcast_to(event, client_id) { + warn!("Failed to send task status update to the client (ID={client_id}): {e:?}"); + } + }; + } }; update_result } @@ -216,11 +227,22 @@ impl RpcTaskManager { fn update_in_progress_status(&mut self, task_id: TaskId, status: Task::InProgressStatus) -> RpcTaskResult<()> { match self.tasks.remove(&task_id) { - Some(TaskStatusExt::InProgress { abort_handle, .. }) - | Some(TaskStatusExt::Awaiting { abort_handle, .. }) => { + Some(TaskStatusExt::InProgress { + abort_handle, + client_id, + .. + }) + | Some(TaskStatusExt::Awaiting { + abort_handle, + client_id, + .. + }) => { // Insert new in-progress status to the tasks container. - self.tasks - .insert(task_id, TaskStatusExt::InProgress { status, abort_handle }); + self.tasks.insert(task_id, TaskStatusExt::InProgress { + status, + abort_handle, + client_id, + }); Ok(()) }, Some(cancelling @ TaskStatusExt::Cancelling { .. }) => { @@ -247,13 +269,15 @@ impl RpcTaskManager { Some(TaskStatusExt::InProgress { status: next_in_progress_status, abort_handle, + client_id, }) => { // Insert new awaiting status to the tasks container. self.tasks.insert(task_id, TaskStatusExt::Awaiting { status, - abort_handle, action_sender, next_in_progress_status, + abort_handle, + client_id, }); Ok(()) }, @@ -279,8 +303,9 @@ impl RpcTaskManager { match self.tasks.remove(&task_id) { Some(TaskStatusExt::Awaiting { action_sender, - abort_handle, next_in_progress_status: status, + abort_handle, + client_id, .. }) => { let result = action_sender @@ -288,8 +313,11 @@ impl RpcTaskManager { // The task seems to be canceled/aborted for some reason. .map_to_mm(|_user_action| RpcTaskError::Cancelled); // Insert new in-progress status to the tasks container. - self.tasks - .insert(task_id, TaskStatusExt::InProgress { status, abort_handle }); + self.tasks.insert(task_id, TaskStatusExt::InProgress { + status, + abort_handle, + client_id, + }); result }, Some(unexpected) => { @@ -318,12 +346,16 @@ enum TaskStatusExt { InProgress { status: Task::InProgressStatus, abort_handle: TaskAbortHandle, + /// The ID of the client requesting the task. To stream out the updates & results for them. + client_id: Option, }, Awaiting { status: Task::AwaitingStatus, action_sender: UserActionSender, - abort_handle: TaskAbortHandle, next_in_progress_status: Task::InProgressStatus, + abort_handle: TaskAbortHandle, + /// The ID of the client requesting the task. To stream out the updates & results for them. + client_id: Option, }, /// `Cancelling` status is set on [`RpcTaskManager::cancel_task`]. /// This status is used to save the task state before it's actually canceled on [`RpcTaskHandle::on_canceled`], diff --git a/mm2src/rpc_task/src/task.rs b/mm2src/rpc_task/src/task.rs index 3ec372041a..c66fb4cef0 100644 --- a/mm2src/rpc_task/src/task.rs +++ b/mm2src/rpc_task/src/task.rs @@ -15,6 +15,11 @@ pub trait RpcTaskTypes { pub trait RpcTask: RpcTaskTypes + Sized + Send + 'static { fn initial_status(&self) -> Self::InProgressStatus; + /// Returns the ID of the client that initiated/requesting the task. + /// + /// This is related to event streaming and is used to identify the client to whom the updates shall be sent. + fn client_id(&self) -> Option; + /// The method is invoked when the task has been cancelled. async fn cancel(self); From 5d903876129e654d3c8431e3e3b06500a7eada4c Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 11 Sep 2024 14:20:27 +0300 Subject: [PATCH 094/128] not important; fix test compilation - WithdrawRequest --- mm2src/coins/eth/eth_tests.rs | 10 +--- mm2src/coins/lp_coins.rs | 14 ++---- mm2src/coins/qrc20/qrc20_tests.rs | 11 +---- mm2src/coins/solana/solana_tests.rs | 29 ++---------- mm2src/coins/solana/spl_tests.rs | 6 +-- mm2src/coins/utxo/utxo_tests.rs | 71 ++++++----------------------- 6 files changed, 26 insertions(+), 115 deletions(-) diff --git a/mm2src/coins/eth/eth_tests.rs b/mm2src/coins/eth/eth_tests.rs index 9332594931..e96b998d19 100644 --- a/mm2src/coins/eth/eth_tests.rs +++ b/mm2src/coins/eth/eth_tests.rs @@ -211,16 +211,13 @@ fn test_withdraw_impl_manual_fee() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: "0x7Bc1bBDD6A0a722fC9bffC49c921B685ECB84b94".to_string(), coin: "ETH".to_string(), - max: false, fee: Some(WithdrawFee::EthGas { gas: gas_limit::ETH_MAX_TRADE_GAS, gas_price: 1.into(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; coin.get_balance().wait().unwrap(); @@ -260,16 +257,13 @@ fn test_withdraw_impl_fee_details() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: "0x7Bc1bBDD6A0a722fC9bffC49c921B685ECB84b94".to_string(), coin: "JST".to_string(), - max: false, fee: Some(WithdrawFee::EthGas { gas: gas_limit::ETH_MAX_TRADE_GAS, gas_price: 1.into(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; coin.get_balance().wait().unwrap(); diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 7a631d2253..eb212f82b8 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -2160,7 +2160,7 @@ pub trait GetWithdrawSenderAddress { /// Instead, accept a generic type from withdraw implementations. /// This way we won't have to update the payload for every platform when /// one of them requires specific addition. -#[derive(Clone, Deserialize)] +#[derive(Clone, Default, Deserialize)] pub struct WithdrawRequest { coin: String, from: Option, @@ -2222,15 +2222,9 @@ impl WithdrawRequest { pub fn new_max(coin: String, to: String) -> WithdrawRequest { WithdrawRequest { coin, - from: None, to, - amount: 0.into(), max: true, - fee: None, - memo: None, - ibc_source_channel: None, - #[cfg(target_arch = "wasm32")] - broadcast: false, + ..Default::default() } } } @@ -5691,10 +5685,8 @@ pub mod for_tests { }), to: to.to_owned(), coin: ticker.to_owned(), - max: false, fee, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let init = init_withdraw(ctx.clone(), withdraw_req).await.unwrap(); let timeout = wait_until_ms(150000); diff --git a/mm2src/coins/qrc20/qrc20_tests.rs b/mm2src/coins/qrc20/qrc20_tests.rs index 2caf87c3bf..8ca65f5e49 100644 --- a/mm2src/coins/qrc20/qrc20_tests.rs +++ b/mm2src/coins/qrc20/qrc20_tests.rs @@ -88,13 +88,9 @@ fn test_withdraw_to_p2sh_address_should_fail() { let req = WithdrawRequest { amount: 10.into(), - from: None, to: p2sh_address.to_string(), coin: "QRC20".into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let err = coin.withdraw(req).wait().unwrap_err().into_inner(); let expect = WithdrawError::InvalidAddress("QRC20 can be sent to P2PKH addresses only".to_owned()); @@ -129,16 +125,13 @@ fn test_withdraw_impl_fee_details() { let withdraw_req = WithdrawRequest { amount: 10.into(), - from: None, to: "qHmJ3KA6ZAjR9wGjpFASn4gtUSeFAqdZgs".into(), coin: "QRC20".into(), - max: false, fee: Some(WithdrawFee::Qrc20Gas { gas_limit: 2_500_000, gas_price: 40, }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); diff --git a/mm2src/coins/solana/solana_tests.rs b/mm2src/coins/solana/solana_tests.rs index fe2b104293..1164e01d80 100644 --- a/mm2src/coins/solana/solana_tests.rs +++ b/mm2src/coins/solana/solana_tests.rs @@ -160,13 +160,9 @@ fn solana_transaction_simulations() { sol_coin .withdraw(WithdrawRequest { coin: "SOL".to_string(), - from: None, to: sol_coin.my_address.clone(), amount: request_amount.clone(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ) @@ -189,13 +185,9 @@ fn solana_transaction_zero_balance() { sol_coin .withdraw(WithdrawRequest { coin: "SOL".to_string(), - from: None, to: sol_coin.my_address.clone(), amount: BigDecimal::from_str("0.000001").unwrap(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ); @@ -219,13 +211,9 @@ fn solana_transaction_simulations_not_enough_for_fees() { sol_coin .withdraw(WithdrawRequest { coin: "SOL".to_string(), - from: None, to: sol_coin.my_address.clone(), amount: BigDecimal::from(1), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ); @@ -254,13 +242,10 @@ fn solana_transaction_simulations_max() { sol_coin .withdraw(WithdrawRequest { coin: "SOL".to_string(), - from: None, to: sol_coin.my_address.clone(), amount: BigDecimal::from(0), max: true, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ) @@ -284,13 +269,9 @@ fn solana_test_transactions() { sol_coin .withdraw(WithdrawRequest { coin: "SOL".to_string(), - from: None, to: sol_coin.my_address.clone(), amount: BigDecimal::try_from(0.0001).unwrap(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ) diff --git a/mm2src/coins/solana/spl_tests.rs b/mm2src/coins/solana/spl_tests.rs index 10943e6e33..e4c621473c 100644 --- a/mm2src/coins/solana/spl_tests.rs +++ b/mm2src/coins/solana/spl_tests.rs @@ -107,13 +107,9 @@ fn test_spl_transactions() { usdc_sol_coin .withdraw(WithdrawRequest { coin: "USDC".to_string(), - from: None, to: "AYJmtzc9D4KU6xsDzhKShFyYKUNXY622j9QoQEo4LfpX".to_string(), amount: withdraw_amount.clone(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }) .compat(), ) diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index aec14d3dec..8497467159 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -596,15 +596,12 @@ fn test_withdraw_impl_set_fixed_fee() { let withdraw_req = WithdrawRequest { amount: 1u64.into(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), - max: false, fee: Some(WithdrawFee::UtxoFixed { amount: "0.1".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; let expected = Some( UtxoFeeDetails { @@ -642,15 +639,12 @@ fn test_withdraw_impl_sat_per_kb_fee() { let withdraw_req = WithdrawRequest { amount: 1u64.into(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), - max: false, fee: Some(WithdrawFee::UtxoPerKbyte { amount: "0.1".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; // The resulting transaction size might be 244 or 245 bytes depending on signature size // MM2 always expects the worst case during fee calculation @@ -691,15 +685,12 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max() { let withdraw_req = WithdrawRequest { amount: "9.9789".parse().unwrap(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), - max: false, fee: Some(WithdrawFee::UtxoPerKbyte { amount: "0.1".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); // The resulting transaction size might be 210 or 211 bytes depending on signature size @@ -742,15 +733,12 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_equal_to_max_dust_included_to_fee() let withdraw_req = WithdrawRequest { amount: "9.9789".parse().unwrap(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), - max: false, fee: Some(WithdrawFee::UtxoPerKbyte { amount: "0.09999999".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); // The resulting transaction size might be 210 or 211 bytes depending on signature size @@ -793,15 +781,12 @@ fn test_withdraw_impl_sat_per_kb_fee_amount_over_max() { let withdraw_req = WithdrawRequest { amount: "9.97939455".parse().unwrap(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), - max: false, fee: Some(WithdrawFee::UtxoPerKbyte { amount: "0.1".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; coin.withdraw(withdraw_req).wait().unwrap_err(); } @@ -831,15 +816,13 @@ fn test_withdraw_impl_sat_per_kb_fee_max() { let withdraw_req = WithdrawRequest { amount: 0u64.into(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: TEST_COIN_NAME.into(), max: true, fee: Some(WithdrawFee::UtxoPerKbyte { amount: "0.1".parse().unwrap(), }), - memo: None, - ibc_source_channel: None, + ..Default::default() }; // The resulting transaction size might be 210 or 211 bytes depending on signature size // MM2 always expects the worst case during fee calculation @@ -897,13 +880,9 @@ fn test_withdraw_kmd_rewards_impl( let withdraw_req = WithdrawRequest { amount: BigDecimal::from_str("0.00001").unwrap(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: "KMD".to_owned(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let expected_fee = TxFeeDetails::Utxo(UtxoFeeDetails { coin: Some("KMD".into()), @@ -976,13 +955,9 @@ fn test_withdraw_rick_rewards_none() { let withdraw_req = WithdrawRequest { amount: BigDecimal::from_str("0.00001").unwrap(), - from: None, to: "RQq6fWoy8aGGMLjvRfMY5mBNVm2RQxJyLa".to_string(), coin: "RICK".to_owned(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let expected_fee = TxFeeDetails::Utxo(UtxoFeeDetails { coin: Some(TEST_COIN_NAME.into()), @@ -3186,13 +3161,9 @@ fn test_withdraw_to_p2pk_fails() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: "03f8f8fa2062590ba9a0a7a86f937de22f540c015864aad35a2a9f6766de906265".to_string(), coin: TEST_COIN_NAME.into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; assert!(matches!( @@ -3241,13 +3212,9 @@ fn test_withdraw_to_p2pkh() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: p2pkh_address.to_string(), coin: TEST_COIN_NAME.into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); let transaction: UtxoTx = deserialize(tx_details.tx.tx_hex().unwrap().as_slice()).unwrap(); @@ -3298,13 +3265,9 @@ fn test_withdraw_to_p2sh() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: p2sh_address.to_string(), coin: TEST_COIN_NAME.into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); let transaction: UtxoTx = deserialize(tx_details.tx.tx_hex().unwrap().as_slice()).unwrap(); @@ -3355,13 +3318,9 @@ fn test_withdraw_to_p2wpkh() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: p2wpkh_address.to_string(), coin: TEST_COIN_NAME.into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); let transaction: UtxoTx = deserialize(tx_details.tx.tx_hex().unwrap().as_slice()).unwrap(); @@ -3403,13 +3362,9 @@ fn test_withdraw_p2pk_balance() { let withdraw_req = WithdrawRequest { amount: 1.into(), - from: None, to: my_p2pkh_address.to_string(), coin: TEST_COIN_NAME.into(), - max: false, - fee: None, - memo: None, - ibc_source_channel: None, + ..Default::default() }; let tx_details = coin.withdraw(withdraw_req).wait().unwrap(); let transaction: UtxoTx = deserialize(tx_details.tx.tx_hex().unwrap().as_slice()).unwrap(); From 5c9dbac8936136a648fad41302ef2dc281ca6f92 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 12 Sep 2024 14:04:54 +0300 Subject: [PATCH 095/128] bch standalone activation via task manager --- mm2src/coins/utxo/bch.rs | 23 ++-- mm2src/coins_activation/src/context.rs | 4 +- mm2src/coins_activation/src/prelude.rs | 5 + .../utxo_activation/init_bch_activation.rs | 116 ++++++++++++++++++ .../src/utxo_activation/mod.rs | 2 + .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 27 ++++ 6 files changed, 165 insertions(+), 12 deletions(-) create mode 100644 mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index bb5768e2b7..ce2ec726f0 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -45,8 +45,8 @@ pub type BchUnspentMap = HashMap; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct BchActivationRequest { #[serde(default)] - allow_slp_unsafe_conf: bool, - bchd_urls: Vec, + pub allow_slp_unsafe_conf: bool, + pub bchd_urls: Vec, #[serde(flatten)] pub utxo_params: UtxoActivationParams, } @@ -162,6 +162,15 @@ impl From for IsSlpUtxoError { } impl BchCoin { + pub fn new(utxo_arc: UtxoArc, slp_addr_prefix: CashAddrPrefix, bchd_urls: Vec) -> Self { + BchCoin { + utxo_arc, + slp_addr_prefix, + bchd_urls, + slp_tokens_infos: Arc::new(Mutex::new(HashMap::new())), + } + } + pub fn slp_prefix(&self) -> &CashAddrPrefix { &self.slp_addr_prefix } pub fn slp_address(&self, address: &Address) -> Result { @@ -631,15 +640,7 @@ pub async fn bch_coin_with_policy( } let bchd_urls = params.bchd_urls; - let slp_tokens_infos = Arc::new(Mutex::new(HashMap::new())); - let constructor = { - move |utxo_arc| BchCoin { - utxo_arc, - slp_addr_prefix: slp_addr_prefix.clone(), - bchd_urls: bchd_urls.clone(), - slp_tokens_infos: slp_tokens_infos.clone(), - } - }; + let constructor = { move |utxo_arc| BchCoin::new(utxo_arc, slp_addr_prefix.clone(), bchd_urls.clone()) }; let coin = try_s!( UtxoArcBuilder::new(ctx, ticker, conf, ¶ms.utxo_params, priv_key_policy, constructor) diff --git a/mm2src/coins_activation/src/context.rs b/mm2src/coins_activation/src/context.rs index b26f24e3b6..d3fe4265cf 100644 --- a/mm2src/coins_activation/src/context.rs +++ b/mm2src/coins_activation/src/context.rs @@ -4,7 +4,7 @@ use crate::init_erc20_token_activation::Erc20TokenTaskManagerShared; use crate::lightning_activation::LightningTaskManagerShared; #[cfg(feature = "enable-sia")] use crate::sia_coin_activation::SiaCoinTaskManagerShared; -use crate::utxo_activation::{QtumTaskManagerShared, UtxoStandardTaskManagerShared}; +use crate::utxo_activation::{BchTaskManagerShared, QtumTaskManagerShared, UtxoStandardTaskManagerShared}; use crate::z_coin_activation::ZcoinTaskManagerShared; use mm2_core::mm_ctx::{from_ctx, MmArc}; use rpc_task::RpcTaskManager; @@ -12,6 +12,7 @@ use std::sync::Arc; pub struct CoinsActivationContext { pub(crate) init_utxo_standard_task_manager: UtxoStandardTaskManagerShared, + pub(crate) init_bch_task_manager: BchTaskManagerShared, pub(crate) init_qtum_task_manager: QtumTaskManagerShared, #[cfg(feature = "enable-sia")] pub(crate) init_sia_task_manager: SiaCoinTaskManagerShared, @@ -30,6 +31,7 @@ impl CoinsActivationContext { #[cfg(feature = "enable-sia")] init_sia_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_utxo_standard_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_bch_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_qtum_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_z_coin_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_eth_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), diff --git a/mm2src/coins_activation/src/prelude.rs b/mm2src/coins_activation/src/prelude.rs index d000170fa3..0fd890fa64 100644 --- a/mm2src/coins_activation/src/prelude.rs +++ b/mm2src/coins_activation/src/prelude.rs @@ -1,5 +1,6 @@ #[cfg(feature = "enable-sia")] use coins::siacoin::SiaCoinActivationParams; +use coins::utxo::bch::BchActivationRequest; use coins::utxo::UtxoActivationParams; use coins::z_coin::ZcoinActivationParams; use coins::{coin_conf, CoinBalance, CoinProtocol, DerivationMethodResponse, MmCoinEnum}; @@ -22,6 +23,10 @@ impl TxHistory for UtxoActivationParams { fn tx_history(&self) -> bool { self.tx_history } } +impl TxHistory for BchActivationRequest { + fn tx_history(&self) -> bool { self.utxo_params.tx_history } +} + #[cfg(feature = "enable-sia")] impl TxHistory for SiaCoinActivationParams { fn tx_history(&self) -> bool { self.tx_history } diff --git a/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs new file mode 100644 index 0000000000..07574944ba --- /dev/null +++ b/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs @@ -0,0 +1,116 @@ +use crate::context::CoinsActivationContext; +use crate::prelude::TryFromCoinProtocol; +use crate::standalone_coin::{InitStandaloneCoinActivationOps, InitStandaloneCoinTaskHandleShared, + InitStandaloneCoinTaskManagerShared}; +use crate::utxo_activation::common_impl::{get_activation_result, priv_key_build_policy, + start_history_background_fetching}; +use crate::utxo_activation::init_utxo_standard_activation_error::InitUtxoStandardError; +use crate::utxo_activation::init_utxo_standard_statuses::{UtxoStandardAwaitingStatus, UtxoStandardInProgressStatus, + UtxoStandardUserAction}; +use crate::utxo_activation::utxo_standard_activation_result::UtxoStandardActivationResult; +use async_trait::async_trait; +use coins::my_tx_history_v2::TxHistoryStorage; +use coins::utxo::bch::CashAddrPrefix; +use coins::utxo::bch::{BchActivationRequest, BchCoin}; +use coins::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilder}; +use coins::CoinProtocol; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::prelude::*; +use mm2_metrics::MetricsArc; +use mm2_number::BigDecimal; +use serde_json::Value as Json; +use std::collections::HashMap; +use std::str::FromStr; + +pub type BchTaskManagerShared = InitStandaloneCoinTaskManagerShared; +pub type BchRpcTaskHandleShared = InitStandaloneCoinTaskHandleShared; + +#[derive(Clone)] +pub struct BchProtocolInfo { + slp_prefix: String, +} + +impl TryFromCoinProtocol for BchProtocolInfo { + fn try_from_coin_protocol(proto: CoinProtocol) -> Result> + where + Self: Sized, + { + match proto { + CoinProtocol::BCH { slp_prefix } => Ok(BchProtocolInfo { slp_prefix }), + protocol => MmError::err(protocol), + } + } +} + +#[async_trait] +impl InitStandaloneCoinActivationOps for BchCoin { + type ActivationRequest = BchActivationRequest; + type StandaloneProtocol = BchProtocolInfo; + type ActivationResult = UtxoStandardActivationResult; + type ActivationError = InitUtxoStandardError; + type InProgressStatus = UtxoStandardInProgressStatus; + type AwaitingStatus = UtxoStandardAwaitingStatus; + type UserAction = UtxoStandardUserAction; + + fn rpc_task_manager(activation_ctx: &CoinsActivationContext) -> &BchTaskManagerShared { + &activation_ctx.init_bch_task_manager + } + + async fn init_standalone_coin( + ctx: MmArc, + ticker: String, + coin_conf: Json, + activation_request: &Self::ActivationRequest, + protocol_info: Self::StandaloneProtocol, + _task_handle: BchRpcTaskHandleShared, + ) -> Result> { + if activation_request.bchd_urls.is_empty() && !activation_request.allow_slp_unsafe_conf { + Err(InitUtxoStandardError::CoinCreationError { + ticker: ticker.clone(), + error: "Using empty bchd_urls is unsafe for SLP users!".into(), + })?; + } + let prefix = CashAddrPrefix::from_str(&protocol_info.slp_prefix).map_err(|e| { + InitUtxoStandardError::CoinCreationError { + ticker: ticker.clone(), + error: format!("Couldn't parse cash address prefix: {e:?}"), + } + })?; + let priv_key_policy = priv_key_build_policy(&ctx, activation_request.utxo_params.priv_key_policy)?; + + let bchd_urls = activation_request.bchd_urls.clone(); + let constructor = { move |utxo_arc| BchCoin::new(utxo_arc, prefix.clone(), bchd_urls.clone()) }; + + let coin = UtxoArcBuilder::new( + &ctx, + &ticker, + &coin_conf, + &activation_request.utxo_params, + priv_key_policy, + constructor, + ) + .build() + .await + .mm_err(|e| InitUtxoStandardError::from_build_err(e, ticker.clone()))?; + + Ok(coin) + } + + async fn get_activation_result( + &self, + ctx: MmArc, + task_handle: BchRpcTaskHandleShared, + activation_request: &Self::ActivationRequest, + ) -> MmResult { + get_activation_result(&ctx, self, task_handle, &activation_request.utxo_params).await + } + + fn start_history_background_fetching( + &self, + metrics: MetricsArc, + storage: impl TxHistoryStorage, + current_balances: HashMap, + ) { + start_history_background_fetching(self.clone(), metrics, storage, current_balances) + } +} diff --git a/mm2src/coins_activation/src/utxo_activation/mod.rs b/mm2src/coins_activation/src/utxo_activation/mod.rs index 5ef6021199..52244ed2dc 100644 --- a/mm2src/coins_activation/src/utxo_activation/mod.rs +++ b/mm2src/coins_activation/src/utxo_activation/mod.rs @@ -1,10 +1,12 @@ mod common_impl; +mod init_bch_activation; mod init_qtum_activation; mod init_utxo_standard_activation; mod init_utxo_standard_activation_error; mod init_utxo_standard_statuses; mod utxo_standard_activation_result; +pub use init_bch_activation::BchTaskManagerShared; pub use init_qtum_activation::QtumTaskManagerShared; pub use init_utxo_standard_activation::UtxoStandardTaskManagerShared; diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 16b2cba07c..2195962579 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -256,6 +256,10 @@ async fn rpc_task_dispatcher( "create_new_account::init" => handle_mmrpc(ctx, request, init_create_new_account).await, "create_new_account::status" => handle_mmrpc(ctx, request, init_create_new_account_status).await, "create_new_account::user_action" => handle_mmrpc(ctx, request, init_create_new_account_user_action).await, + "enable_bch::cancel" => handle_mmrpc(ctx, request, cancel_init_standalone_coin::).await, + "enable_bch::init" => handle_mmrpc(ctx, request, init_standalone_coin::).await, + "enable_bch::status" => handle_mmrpc(ctx, request, init_standalone_coin_status::).await, + "enable_bch::user_action" => handle_mmrpc(ctx, request, init_standalone_coin_user_action::).await, "enable_qtum::cancel" => handle_mmrpc(ctx, request, cancel_init_standalone_coin::).await, "enable_qtum::init" => handle_mmrpc(ctx, request, init_standalone_coin::).await, "enable_qtum::status" => handle_mmrpc(ctx, request, init_standalone_coin_status::).await, @@ -276,6 +280,29 @@ async fn rpc_task_dispatcher( "enable_erc20::init" => handle_mmrpc(ctx, request, init_token::).await, "enable_erc20::status" => handle_mmrpc(ctx, request, init_token_status::).await, "enable_erc20::user_action" => handle_mmrpc(ctx, request, init_token_user_action::).await, + // // FIXME: tendermint with tokens + // "enable_tendermint::cancel" => { + // handle_mmrpc(ctx, request, cancel_init_platform_coin_with_tokens::).await + // }, + // "enable_tendermint::init" => handle_mmrpc(ctx, request, init_platform_coin_with_tokens::).await, + // "enable_tendermint::status" => { + // handle_mmrpc(ctx, request, init_platform_coin_with_tokens_status::).await + // }, + // "enable_tendermint::user_action" => { + // handle_mmrpc( + // ctx, + // request, + // init_platform_coin_with_tokens_user_action::, + // ) + // .await + // }, + // // FIXME: tendermint tokens + // "enable_tendermint_token::cancel" => handle_mmrpc(ctx, request, cancel_init_token::).await, + // "enable_tendermint_token::init" => handle_mmrpc(ctx, request, init_token::).await, + // "enable_tendermint_token::status" => handle_mmrpc(ctx, request, init_token_status::).await, + // "enable_tendermint_token::user_action" => { + // handle_mmrpc(ctx, request, init_token_user_action::).await + // }, "get_new_address::cancel" => handle_mmrpc(ctx, request, cancel_get_new_address).await, "get_new_address::init" => handle_mmrpc(ctx, request, init_get_new_address).await, "get_new_address::status" => handle_mmrpc(ctx, request, init_get_new_address_status).await, From 0ea63672bef69053baa08b1c64f710bf7345a156 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 12 Sep 2024 14:36:27 +0300 Subject: [PATCH 096/128] tendermint coin with-tokens-activation via task manager --- mm2src/coins_activation/src/context.rs | 3 ++ .../src/tendermint_with_assets_activation.rs | 8 ++--- .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 31 +++++++++---------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/mm2src/coins_activation/src/context.rs b/mm2src/coins_activation/src/context.rs index d3fe4265cf..dca33f40c8 100644 --- a/mm2src/coins_activation/src/context.rs +++ b/mm2src/coins_activation/src/context.rs @@ -4,6 +4,7 @@ use crate::init_erc20_token_activation::Erc20TokenTaskManagerShared; use crate::lightning_activation::LightningTaskManagerShared; #[cfg(feature = "enable-sia")] use crate::sia_coin_activation::SiaCoinTaskManagerShared; +use crate::tendermint_with_assets_activation::TendermintCoinTaskManagerShared; use crate::utxo_activation::{BchTaskManagerShared, QtumTaskManagerShared, UtxoStandardTaskManagerShared}; use crate::z_coin_activation::ZcoinTaskManagerShared; use mm2_core::mm_ctx::{from_ctx, MmArc}; @@ -19,6 +20,7 @@ pub struct CoinsActivationContext { pub(crate) init_z_coin_task_manager: ZcoinTaskManagerShared, pub(crate) init_eth_task_manager: EthTaskManagerShared, pub(crate) init_erc20_token_task_manager: Erc20TokenTaskManagerShared, + pub(crate) init_tendermint_coin_task_manager: TendermintCoinTaskManagerShared, #[cfg(not(target_arch = "wasm32"))] pub(crate) init_lightning_task_manager: LightningTaskManagerShared, } @@ -36,6 +38,7 @@ impl CoinsActivationContext { init_z_coin_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_eth_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), init_erc20_token_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), + init_tendermint_coin_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), #[cfg(not(target_arch = "wasm32"))] init_lightning_task_manager: RpcTaskManager::new_shared(ctx.event_stream_manager.clone()), }) diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index 65dce58ca6..b9c0673d43 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -27,6 +27,8 @@ use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value as Json; use std::collections::{HashMap, HashSet}; +pub type TendermintCoinTaskManagerShared = InitPlatformCoinWithTokensTaskManagerShared; + impl TokenOf for TendermintToken { type PlatformCoin = TendermintCoin; } @@ -367,9 +369,7 @@ impl PlatformCoinWithTokensActivationOps for TendermintCoin { self.spawner().spawn_with_settings(fut, settings); } - fn rpc_task_manager( - _activation_ctx: &CoinsActivationContext, - ) -> &InitPlatformCoinWithTokensTaskManagerShared { - unimplemented!() + fn rpc_task_manager(activation_ctx: &CoinsActivationContext) -> &TendermintCoinTaskManagerShared { + &activation_ctx.init_tendermint_coin_task_manager } } diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 2195962579..90d109604b 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -280,22 +280,21 @@ async fn rpc_task_dispatcher( "enable_erc20::init" => handle_mmrpc(ctx, request, init_token::).await, "enable_erc20::status" => handle_mmrpc(ctx, request, init_token_status::).await, "enable_erc20::user_action" => handle_mmrpc(ctx, request, init_token_user_action::).await, - // // FIXME: tendermint with tokens - // "enable_tendermint::cancel" => { - // handle_mmrpc(ctx, request, cancel_init_platform_coin_with_tokens::).await - // }, - // "enable_tendermint::init" => handle_mmrpc(ctx, request, init_platform_coin_with_tokens::).await, - // "enable_tendermint::status" => { - // handle_mmrpc(ctx, request, init_platform_coin_with_tokens_status::).await - // }, - // "enable_tendermint::user_action" => { - // handle_mmrpc( - // ctx, - // request, - // init_platform_coin_with_tokens_user_action::, - // ) - // .await - // }, + "enable_tendermint::cancel" => { + handle_mmrpc(ctx, request, cancel_init_platform_coin_with_tokens::).await + }, + "enable_tendermint::init" => handle_mmrpc(ctx, request, init_platform_coin_with_tokens::).await, + "enable_tendermint::status" => { + handle_mmrpc(ctx, request, init_platform_coin_with_tokens_status::).await + }, + "enable_tendermint::user_action" => { + handle_mmrpc( + ctx, + request, + init_platform_coin_with_tokens_user_action::, + ) + .await + }, // // FIXME: tendermint tokens // "enable_tendermint_token::cancel" => handle_mmrpc(ctx, request, cancel_init_token::).await, // "enable_tendermint_token::init" => handle_mmrpc(ctx, request, init_token::).await, From c05140094a6f36c04a93b16706092b2e33fb1cc7 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sun, 15 Sep 2024 19:20:15 +0300 Subject: [PATCH 097/128] move tendermint tokens enabler to a todo this was too involved into HD wallet stuff and couldn't add it easily. Defering this as it will take some time/getting familiar with some tendermint & HD wallet constructs --- mm2src/mm2_core/src/data_asker.rs | 1 - mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mm2src/mm2_core/src/data_asker.rs b/mm2src/mm2_core/src/data_asker.rs index 9a0c64ef90..edcff694d2 100644 --- a/mm2src/mm2_core/src/data_asker.rs +++ b/mm2src/mm2_core/src/data_asker.rs @@ -70,7 +70,6 @@ impl MmCtx { "data": data }); - // FIXME: This event will be broadcasted to all the clients. We should only hand it to one client? self.event_stream_manager .broadcast_all(Event::new(format!("{EVENT_NAME}:{data_type}"), input)); diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 90d109604b..90cc69f5fc 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -295,7 +295,7 @@ async fn rpc_task_dispatcher( ) .await }, - // // FIXME: tendermint tokens + // // TODO: tendermint tokens // "enable_tendermint_token::cancel" => handle_mmrpc(ctx, request, cancel_init_token::).await, // "enable_tendermint_token::init" => handle_mmrpc(ctx, request, init_token::).await, // "enable_tendermint_token::status" => handle_mmrpc(ctx, request, init_token_status::).await, From 4ca4d693783fc2d28a1fb96b553800deba8a097c Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 30 Sep 2024 20:21:34 +0200 Subject: [PATCH 098/128] try_new -> new & let serde do the erroring job --- mm2src/coins/eth/fee_estimation/eth_fee_events.rs | 11 +++-------- mm2src/mm2_main/src/heartbeat_event.rs | 9 ++------- .../src/rpc/streaming_activations/fee_estimation.rs | 9 +++------ .../src/rpc/streaming_activations/heartbeat.rs | 9 +++------ .../mm2_main/src/rpc/streaming_activations/network.rs | 9 +++------ mm2src/mm2_net/src/network_event.rs | 11 +++-------- 6 files changed, 17 insertions(+), 41 deletions(-) diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index cda6e5e34d..dfe804ac02 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -6,7 +6,6 @@ use async_trait::async_trait; use futures::channel::oneshot; use instant::Instant; use serde::Deserialize; -use serde_json::Value as Json; use std::convert::TryFrom; #[derive(Deserialize)] @@ -14,14 +13,14 @@ use std::convert::TryFrom; /// Types of estimators available. /// Simple - simple internal gas price estimator based on historical data. /// Provider - gas price estimator using external provider (using gas api). -enum EstimatorType { +pub enum EstimatorType { Simple, Provider, } #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] -struct EthFeeStreamingConfig { +pub struct EthFeeStreamingConfig { /// The time in seconds to wait before re-estimating the gas fees. pub estimate_every: f64, /// The type of the estimator to use. @@ -43,11 +42,7 @@ pub struct EthFeeEventStreamer { } impl EthFeeEventStreamer { - pub fn try_new(config: Option, coin: EthCoin) -> serde_json::Result { - let config = config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?; - - Ok(Self { config, coin }) - } + pub fn new(config: EthFeeStreamingConfig, coin: EthCoin) -> Self { Self { config, coin } } } #[async_trait] diff --git a/mm2src/mm2_main/src/heartbeat_event.rs b/mm2src/mm2_main/src/heartbeat_event.rs index 21175aafbf..a2c46f2fb6 100644 --- a/mm2src/mm2_main/src/heartbeat_event.rs +++ b/mm2src/mm2_main/src/heartbeat_event.rs @@ -3,11 +3,10 @@ use common::executor::Timer; use futures::channel::oneshot; use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use serde::Deserialize; -use serde_json::Value as Json; #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] -struct HeartbeatEventConfig { +pub struct HeartbeatEventConfig { /// The time in seconds to wait before sending another ping event. pub stream_interval_seconds: f64, } @@ -25,11 +24,7 @@ pub struct HeartbeatEvent { } impl HeartbeatEvent { - pub fn try_new(config: Option) -> serde_json::Result { - Ok(Self { - config: config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?, - }) - } + pub fn new(config: HeartbeatEventConfig) -> Self { Self { config } } } #[async_trait] diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index b9350739bb..6b80f4040d 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -1,20 +1,18 @@ //! RPC activation and deactivation for different fee estimation streamers. use super::EnableStreamingResponse; -use coins::eth::fee_estimation::eth_fee_events::EthFeeEventStreamer; +use coins::eth::fee_estimation::eth_fee_events::{EthFeeEventStreamer, EthFeeStreamingConfig}; use coins::{lp_coinfind, MmCoin, MmCoinEnum}; use common::HttpStatusCode; use http::StatusCode; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; -use serde_json::Value as Json; - #[derive(Deserialize)] pub struct EnableFeeStreamingRequest { pub client_id: u64, pub coin: String, - pub config: Option, + pub config: EthFeeStreamingConfig, } #[derive(Display, Serialize, SerializeErrorType)] @@ -48,8 +46,7 @@ pub async fn enable_fee_estimation( match coin { MmCoinEnum::EthCoin(coin) => { - let eth_fee_estimator_streamer = EthFeeEventStreamer::try_new(req.config, coin.clone()) - .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}")))?; + let eth_fee_estimator_streamer = EthFeeEventStreamer::new(req.config, coin.clone()); ctx.event_stream_manager .add(req.client_id, eth_fee_estimator_streamer, coin.spawner()) .await diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index 12d34637e2..2169c3dcaa 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -1,18 +1,16 @@ //! RPC activation and deactivation for the heartbeats. use super::EnableStreamingResponse; -use crate::heartbeat_event::HeartbeatEvent; +use crate::heartbeat_event::{HeartbeatEvent, HeartbeatEventConfig}; use common::HttpStatusCode; use http::StatusCode; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; -use serde_json::Value as Json; - #[derive(Deserialize)] pub struct EnableHeartbeatRequest { pub client_id: u64, - pub config: Option, + pub config: HeartbeatEventConfig, } #[derive(Display, Serialize, SerializeErrorType)] @@ -29,8 +27,7 @@ pub async fn enable_heartbeat( ctx: MmArc, req: EnableHeartbeatRequest, ) -> MmResult { - let heartbeat_streamer = - HeartbeatEvent::try_new(req.config).map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}")))?; + let heartbeat_streamer = HeartbeatEvent::new(req.config); ctx.event_stream_manager .add(req.client_id, heartbeat_streamer, ctx.spawner()) .await diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs index b32234ec1b..4310476828 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -5,14 +5,12 @@ use common::HttpStatusCode; use http::StatusCode; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; -use mm2_net::network_event::NetworkEvent; - -use serde_json::Value as Json; +use mm2_net::network_event::{NetworkEvent, NetworkEventConfig}; #[derive(Deserialize)] pub struct EnableNetworkStreamingRequest { pub client_id: u64, - pub config: Option, + pub config: NetworkEventConfig, } #[derive(Display, Serialize, SerializeErrorType)] @@ -29,8 +27,7 @@ pub async fn enable_network( ctx: MmArc, req: EnableNetworkStreamingRequest, ) -> MmResult { - let network_steamer = NetworkEvent::try_new(req.config, ctx.clone()) - .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}")))?; + let network_steamer = NetworkEvent::new(req.config, ctx.clone()); ctx.event_stream_manager .add(req.client_id, network_steamer, ctx.spawner()) .await diff --git a/mm2src/mm2_net/src/network_event.rs b/mm2src/mm2_net/src/network_event.rs index 90ad0b7bea..acba300a2f 100644 --- a/mm2src/mm2_net/src/network_event.rs +++ b/mm2src/mm2_net/src/network_event.rs @@ -7,11 +7,11 @@ use mm2_libp2p::behaviours::atomicdex; use async_trait::async_trait; use futures::channel::oneshot; use serde::Deserialize; -use serde_json::{json, Value as Json}; +use serde_json::json; #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] -struct NetworkEventConfig { +pub struct NetworkEventConfig { /// The time in seconds to wait after sending network info before sending another one. pub stream_interval_seconds: f64, /// Always (force) send network info data, even if it's the same as the previous one sent. @@ -33,12 +33,7 @@ pub struct NetworkEvent { } impl NetworkEvent { - pub fn try_new(config: Option, ctx: MmArc) -> serde_json::Result { - Ok(Self { - config: config.map(serde_json::from_value).unwrap_or(Ok(Default::default()))?, - ctx, - }) - } + pub fn new(config: NetworkEventConfig, ctx: MmArc) -> Self { Self { config, ctx } } } #[async_trait] From 471c5dbca70563e10f589747b41802b3801114a8 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 30 Sep 2024 20:35:04 +0200 Subject: [PATCH 099/128] keep balance error wrapped in MmErrro --- mm2src/coins/eth/eth_balance_events.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/eth/eth_balance_events.rs b/mm2src/coins/eth/eth_balance_events.rs index 1f3249b7e8..b62bd62ed1 100644 --- a/mm2src/coins/eth/eth_balance_events.rs +++ b/mm2src/coins/eth/eth_balance_events.rs @@ -2,6 +2,7 @@ use super::EthCoin; use crate::{eth::{u256_to_big_decimal, Erc20TokenInfo}, BalanceError, CoinWithDerivationMethod}; use common::{executor::Timer, log, Future01CompatExt}; +use mm2_err_handle::prelude::MmError; use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; use mm2_number::BigDecimal; @@ -55,7 +56,7 @@ struct BalanceData { struct BalanceFetchError { ticker: String, address: String, - error: BalanceError, + error: MmError, } type BalanceResult = Result; @@ -111,7 +112,7 @@ async fn fetch_balance( .map_err(|error| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error: error.into_inner(), + error, })?, coin.decimals, ) @@ -122,7 +123,7 @@ async fn fetch_balance( .map_err(|error| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error: error.into_inner(), + error, })?, info.decimals, ) @@ -131,7 +132,7 @@ async fn fetch_balance( let balance_as_big_decimal = u256_to_big_decimal(balance_as_u256, decimals).map_err(|e| BalanceFetchError { ticker: token_ticker.clone(), address: address.to_string(), - error: e.into_inner().into(), + error: e.into(), })?; Ok(BalanceData { From d387d1bfc11a1f0d150402f6c66a0210a7105b44 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 2 Oct 2024 08:11:48 +0200 Subject: [PATCH 100/128] re-introduce get_eth_estimated_fee_per_gas rpc but without the start & stop rpcs. this means if the client wants to use this rpc they should implement the caching themselves and not call this rpc more than necessary. --- .../eth/fee_estimation/eth_fee_events.rs | 86 +------------------ mm2src/coins/eth/fee_estimation/mod.rs | 2 + mm2src/coins/eth/fee_estimation/rpc.rs | 57 ++++++++++++ mm2src/coins/eth/fee_estimation/ser.rs | 80 +++++++++++++++++ .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 2 + 5 files changed, 143 insertions(+), 84 deletions(-) create mode 100644 mm2src/coins/eth/fee_estimation/rpc.rs create mode 100644 mm2src/coins/eth/fee_estimation/ser.rs diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index dfe804ac02..852988ef9e 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -1,3 +1,4 @@ +use super::ser::FeePerGasEstimated; use crate::eth::EthCoin; use common::executor::Timer; use mm2_event_stream::{Broadcaster, Event, EventStreamer, NoDataIn, StreamHandlerInput}; @@ -68,7 +69,7 @@ impl EventStreamer for EthFeeEventStreamer { .coin .get_eip1559_gas_fee(use_simple) .await - .map(serialized::FeePerGasEstimated::try_from) + .map(FeePerGasEstimated::try_from) { Ok(Ok(fee)) => { let fee = serde_json::to_value(fee).expect("Serialization shouldn't fail"); @@ -90,86 +91,3 @@ impl EventStreamer for EthFeeEventStreamer { } } } - -/// Serializable version of fee estimation data. -mod serialized { - use crate::eth::fee_estimation::eip1559; - use crate::{wei_to_gwei_decimal, NumConversError}; - use mm2_err_handle::mm_error::MmError; - use mm2_number::BigDecimal; - - use std::convert::TryFrom; - - /// Estimated fee per gas units - #[derive(Serialize)] - pub enum EstimationUnits { - Gwei, - } - - /// Priority level estimated max fee per gas - #[derive(Serialize)] - pub struct FeePerGasLevel { - /// estimated max priority tip fee per gas in gwei - pub max_priority_fee_per_gas: BigDecimal, - /// estimated max fee per gas in gwei - pub max_fee_per_gas: BigDecimal, - /// estimated transaction min wait time in mempool in ms for this priority level - pub min_wait_time: Option, - /// estimated transaction max wait time in mempool in ms for this priority level - pub max_wait_time: Option, - } - - /// External struct for estimated fee per gas for several priority levels, in gwei - /// low/medium/high levels are supported - #[derive(Serialize)] - pub struct FeePerGasEstimated { - /// base fee for the next block in gwei - pub base_fee: BigDecimal, - /// estimated low priority fee - pub low: FeePerGasLevel, - /// estimated medium priority fee - pub medium: FeePerGasLevel, - /// estimated high priority fee - pub high: FeePerGasLevel, - /// which estimator used - pub source: String, - /// base trend (up or down) - pub base_fee_trend: String, - /// priority trend (up or down) - pub priority_fee_trend: String, - /// fee units - pub units: EstimationUnits, - } - - impl TryFrom for FeePerGasEstimated { - type Error = MmError; - - fn try_from(fees: eip1559::FeePerGasEstimated) -> Result { - Ok(Self { - base_fee: wei_to_gwei_decimal!(fees.base_fee)?, - low: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_priority_fee_per_gas)?, - min_wait_time: fees.low.min_wait_time, - max_wait_time: fees.low.max_wait_time, - }, - medium: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_priority_fee_per_gas)?, - min_wait_time: fees.medium.min_wait_time, - max_wait_time: fees.medium.max_wait_time, - }, - high: FeePerGasLevel { - max_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_fee_per_gas)?, - max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_priority_fee_per_gas)?, - min_wait_time: fees.high.min_wait_time, - max_wait_time: fees.high.max_wait_time, - }, - source: fees.source.to_string(), - base_fee_trend: fees.base_fee_trend, - priority_fee_trend: fees.priority_fee_trend, - units: EstimationUnits::Gwei, - }) - } - } -} diff --git a/mm2src/coins/eth/fee_estimation/mod.rs b/mm2src/coins/eth/fee_estimation/mod.rs index f519e95369..ffe9683acd 100644 --- a/mm2src/coins/eth/fee_estimation/mod.rs +++ b/mm2src/coins/eth/fee_estimation/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod eip1559; pub mod eth_fee_events; +pub mod rpc; +mod ser; diff --git a/mm2src/coins/eth/fee_estimation/rpc.rs b/mm2src/coins/eth/fee_estimation/rpc.rs new file mode 100644 index 0000000000..111d8c7979 --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/rpc.rs @@ -0,0 +1,57 @@ +use super::eth_fee_events::EstimatorType; +use super::ser::FeePerGasEstimated; +use crate::{lp_coinfind, MmCoinEnum}; +use common::HttpStatusCode; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::mm_error::MmResult; + +use http::StatusCode; +use std::convert::TryFrom; + +#[derive(Deserialize)] +pub struct GetFeeEstimationRequest { + coin: String, + estimator_type: EstimatorType, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum GetFeeEstimationRequestError { + CoinNotFound, + Internal(String), + CoinNotSupported, +} + +impl HttpStatusCode for GetFeeEstimationRequestError { + fn status_code(&self) -> StatusCode { + match self { + GetFeeEstimationRequestError::CoinNotFound => StatusCode::NOT_FOUND, + GetFeeEstimationRequestError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + GetFeeEstimationRequestError::CoinNotSupported => StatusCode::NOT_IMPLEMENTED, + } + } +} + +pub async fn get_eth_estimated_fee_per_gas( + ctx: MmArc, + req: GetFeeEstimationRequest, +) -> MmResult { + let coin = lp_coinfind(&ctx, &req.coin) + .await + .map_err(GetFeeEstimationRequestError::Internal)? + .ok_or(GetFeeEstimationRequestError::CoinNotFound)?; + + match coin { + MmCoinEnum::EthCoin(coin) => { + let use_simple = matches!(req.estimator_type, EstimatorType::Simple); + let fee = coin + .get_eip1559_gas_fee(use_simple) + .await + .map_err(|e| GetFeeEstimationRequestError::Internal(e.to_string()))?; + let ser_fee = + FeePerGasEstimated::try_from(fee).map_err(|e| GetFeeEstimationRequestError::Internal(e.to_string()))?; + Ok(ser_fee) + }, + _ => Err(GetFeeEstimationRequestError::CoinNotSupported)?, + } +} diff --git a/mm2src/coins/eth/fee_estimation/ser.rs b/mm2src/coins/eth/fee_estimation/ser.rs new file mode 100644 index 0000000000..62331e0f13 --- /dev/null +++ b/mm2src/coins/eth/fee_estimation/ser.rs @@ -0,0 +1,80 @@ +//! Serializable version of fee estimation data. +use crate::eth::fee_estimation::eip1559; +use crate::{wei_to_gwei_decimal, NumConversError}; +use mm2_err_handle::mm_error::MmError; +use mm2_number::BigDecimal; + +use std::convert::TryFrom; + +/// Estimated fee per gas units +#[derive(Serialize)] +pub enum EstimationUnits { + Gwei, +} + +/// Priority level estimated max fee per gas +#[derive(Serialize)] +pub struct FeePerGasLevel { + /// estimated max priority tip fee per gas in gwei + pub max_priority_fee_per_gas: BigDecimal, + /// estimated max fee per gas in gwei + pub max_fee_per_gas: BigDecimal, + /// estimated transaction min wait time in mempool in ms for this priority level + pub min_wait_time: Option, + /// estimated transaction max wait time in mempool in ms for this priority level + pub max_wait_time: Option, +} + +/// External struct for estimated fee per gas for several priority levels, in gwei +/// low/medium/high levels are supported +#[derive(Serialize)] +pub struct FeePerGasEstimated { + /// base fee for the next block in gwei + pub base_fee: BigDecimal, + /// estimated low priority fee + pub low: FeePerGasLevel, + /// estimated medium priority fee + pub medium: FeePerGasLevel, + /// estimated high priority fee + pub high: FeePerGasLevel, + /// which estimator used + pub source: String, + /// base trend (up or down) + pub base_fee_trend: String, + /// priority trend (up or down) + pub priority_fee_trend: String, + /// fee units + pub units: EstimationUnits, +} + +impl TryFrom for FeePerGasEstimated { + type Error = MmError; + + fn try_from(fees: eip1559::FeePerGasEstimated) -> Result { + Ok(Self { + base_fee: wei_to_gwei_decimal!(fees.base_fee)?, + low: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.low.max_priority_fee_per_gas)?, + min_wait_time: fees.low.min_wait_time, + max_wait_time: fees.low.max_wait_time, + }, + medium: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.medium.max_priority_fee_per_gas)?, + min_wait_time: fees.medium.min_wait_time, + max_wait_time: fees.medium.max_wait_time, + }, + high: FeePerGasLevel { + max_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_fee_per_gas)?, + max_priority_fee_per_gas: wei_to_gwei_decimal!(fees.high.max_priority_fee_per_gas)?, + min_wait_time: fees.high.min_wait_time, + max_wait_time: fees.high.max_wait_time, + }, + source: fees.source.to_string(), + base_fee_trend: fees.base_fee_trend, + priority_fee_trend: fees.priority_fee_trend, + units: EstimationUnits::Gwei, + }) + } +} diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 4611938f75..1cd95714f3 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -12,6 +12,7 @@ use crate::{lp_stats::{add_node_to_version_stat, remove_node_from_version_stat, stop_version_stat_collection, update_version_stat_collection}, lp_swap::{get_locked_amount_rpc, max_maker_vol, recreate_swap_data, trade_preimage_rpc}, rpc::lp_commands::{get_public_key, get_public_key_hash, get_shared_db_id, trezor_connection_status}}; +use coins::eth::fee_estimation::rpc::get_eth_estimated_fee_per_gas; use coins::eth::EthCoin; use coins::my_tx_history_v2::my_tx_history_v2_rpc; use coins::rpc_command::tendermint::{ibc_chains, ibc_transfer_channels}; @@ -219,6 +220,7 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, ibc_chains).await, "ibc_transfer_channels" => handle_mmrpc(ctx, request, ibc_transfer_channels).await, "withdraw_nft" => handle_mmrpc(ctx, request, withdraw_nft).await, + "get_eth_estimated_fee_per_gas" => handle_mmrpc(ctx, request, get_eth_estimated_fee_per_gas).await, "get_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, get_swap_transaction_fee_policy).await, "set_swap_transaction_fee_policy" => handle_mmrpc(ctx, request, set_swap_transaction_fee_policy).await, "send_asked_data" => handle_mmrpc(ctx, request, send_asked_data_rpc).await, From f0f626075f64c6ecc239c2b0c3c3a1205f65b959 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 2 Oct 2024 08:17:35 +0200 Subject: [PATCH 101/128] remove extra line --- mm2src/coins_activation/src/tendermint_with_assets_activation.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs index b9c0673d43..d62c9ebd8b 100644 --- a/mm2src/coins_activation/src/tendermint_with_assets_activation.rs +++ b/mm2src/coins_activation/src/tendermint_with_assets_activation.rs @@ -10,7 +10,6 @@ use crate::prelude::*; use async_trait::async_trait; use coins::hd_wallet::HDPathAccountToAddressId; use coins::my_tx_history_v2::TxHistoryStorage; - use coins::tendermint::tendermint_tx_history_v2::tendermint_history_loop; use coins::tendermint::{tendermint_priv_key_policy, RpcNode, TendermintActivationPolicy, TendermintCoin, TendermintCommons, TendermintConf, TendermintInitError, TendermintInitErrorKind, From fc133d775b2ef19d899ad8517af48366cd8b6968 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 2 Oct 2024 12:16:38 +0200 Subject: [PATCH 102/128] add a fixme: sse endpoint should be authenticated --- mm2src/mm2_main/src/rpc.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/mm2_main/src/rpc.rs b/mm2src/mm2_main/src/rpc.rs index 8588e7a1e1..109396ce8b 100644 --- a/mm2src/mm2_main/src/rpc.rs +++ b/mm2src/mm2_main/src/rpc.rs @@ -344,6 +344,7 @@ pub extern "C" fn spawn_rpc(ctx_h: u32) { let make_svc_fut = move |remote_addr: SocketAddr| async move { Ok::<_, Infallible>(service_fn(move |req: Request| async move { + // FIXME: THIS SHOULD BE AUTHENTICATED!!! if req.uri().path() == SSE_ENDPOINT { let res = handle_sse(req, ctx_h).await?; return Ok::<_, Infallible>(res); From bb90b651ed853c9498aecde2ab25dfc8ac8171d8 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sun, 6 Oct 2024 11:36:32 +0200 Subject: [PATCH 103/128] re-introduce mm2.json event_streaming_configuration grouping relevant options inside it together --- mm2src/mm2_core/src/mm_ctx.rs | 7 ++++++- mm2src/mm2_event_stream/src/configuration.rs | 21 +++++++++++++++++++ mm2src/mm2_event_stream/src/lib.rs | 2 ++ mm2src/mm2_main/src/lp_native_dex.rs | 10 +++++++-- mm2src/mm2_main/src/rpc.rs | 2 +- mm2src/mm2_net/src/event_streaming/mod.rs | 2 ++ .../src/{ => event_streaming}/sse_handler.rs | 13 +++++++----- .../wasm_event_stream.rs | 9 ++------ mm2src/mm2_net/src/lib.rs | 12 ++++------- 9 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 mm2src/mm2_event_stream/src/configuration.rs create mode 100644 mm2src/mm2_net/src/event_streaming/mod.rs rename mm2src/mm2_net/src/{ => event_streaming}/sse_handler.rs (89%) rename mm2src/mm2_net/src/{ => event_streaming}/wasm_event_stream.rs (85%) diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index fa235bd4ae..8ff812caee 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -6,7 +6,7 @@ use common::log::{self, LogLevel, LogOnError, LogState}; use common::{cfg_native, cfg_wasm32, small_rng}; use gstuff::{try_s, Constructible, ERR, ERRL}; use lazy_static::lazy_static; -use mm2_event_stream::StreamingManager; +use mm2_event_stream::{EventStreamingConfiguration, StreamingManager}; use mm2_metrics::{MetricsArc, MetricsOps}; use primitives::hash::H160; use rand::Rng; @@ -332,6 +332,11 @@ impl MmCtx { /// Returns whether node is configured to use [Upgraded Trading Protocol](https://github.com/KomodoPlatform/komodo-defi-framework/issues/1895) pub fn use_trading_proto_v2(&self) -> bool { self.conf["use_trading_proto_v2"].as_bool().unwrap_or_default() } + /// Returns the event streaming configuration in use. + pub fn event_streaming_configuration(&self) -> EventStreamingConfiguration { + serde_json::from_value(self.conf["event_streaming_configuration"].clone()).unwrap_or_default() + } + /// Returns the cloneable `WeakSpawner`. pub fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs new file mode 100644 index 0000000000..532d195e76 --- /dev/null +++ b/mm2src/mm2_event_stream/src/configuration.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(default)] +/// The network-related configuration of the event streaming interface. +// TODO: This better fits in mm2_net but then we would have circular dependency error trying to import it in mm2_core. +pub struct EventStreamingConfiguration { + pub disabled: bool, + pub worker_path: String, + pub access_control_allow_origin: String, +} + +impl Default for EventStreamingConfiguration { + fn default() -> Self { + Self { + disabled: false, + worker_path: "event_streaming_worker.js".to_string(), + access_control_allow_origin: "*".to_string(), + } + } +} diff --git a/mm2src/mm2_event_stream/src/lib.rs b/mm2src/mm2_event_stream/src/lib.rs index f60d2a9d4a..db4587a77a 100644 --- a/mm2src/mm2_event_stream/src/lib.rs +++ b/mm2src/mm2_event_stream/src/lib.rs @@ -1,8 +1,10 @@ +pub mod configuration; pub mod event; pub mod manager; pub mod streamer; // Re-export important types. +pub use configuration::EventStreamingConfiguration; pub use event::Event; pub use manager::{StreamingManager, StreamingManagerError}; pub use streamer::{Broadcaster, EventStreamer, NoDataIn, StreamHandlerInput}; diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index 6b5814676c..a23cad1ad7 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -63,7 +63,7 @@ cfg_native! { #[path = "lp_init/init_hw.rs"] pub mod init_hw; cfg_wasm32! { - use mm2_net::wasm_event_stream::handle_worker_stream; + use mm2_net::event_streaming::wasm_event_stream::handle_worker_stream; #[path = "lp_init/init_metamask.rs"] pub mod init_metamask; @@ -422,7 +422,13 @@ fn migrate_db(ctx: &MmArc) -> MmInitResult<()> { fn migration_1(_ctx: &MmArc) {} #[cfg(target_arch = "wasm32")] -fn init_wasm_event_streaming(ctx: &MmArc) { ctx.spawner().spawn(handle_worker_stream(ctx.clone())); } +fn init_wasm_event_streaming(ctx: &MmArc) { + let event_streaming_config = ctx.event_streaming_configuration(); + if !event_streaming_config.disabled { + ctx.spawner() + .spawn(handle_worker_stream(ctx.clone(), event_streaming_config.worker_path)); + } +} pub async fn lp_init_continue(ctx: MmArc) -> MmInitResult<()> { init_ordermatch_context(&ctx)?; diff --git a/mm2src/mm2_main/src/rpc.rs b/mm2src/mm2_main/src/rpc.rs index 109396ce8b..90d5cab162 100644 --- a/mm2src/mm2_main/src/rpc.rs +++ b/mm2src/mm2_main/src/rpc.rs @@ -37,7 +37,7 @@ use std::net::SocketAddr; cfg_native! { use hyper::{self, Body, Server}; - use mm2_net::sse_handler::{handle_sse, SSE_ENDPOINT}; + use mm2_net::event_streaming::sse_handler::{handle_sse, SSE_ENDPOINT}; } #[path = "rpc/dispatcher/dispatcher.rs"] mod dispatcher; diff --git a/mm2src/mm2_net/src/event_streaming/mod.rs b/mm2src/mm2_net/src/event_streaming/mod.rs new file mode 100644 index 0000000000..001424f5f4 --- /dev/null +++ b/mm2src/mm2_net/src/event_streaming/mod.rs @@ -0,0 +1,2 @@ +#[cfg(not(target_arch = "wasm32"))] pub mod sse_handler; +#[cfg(target_arch = "wasm32")] pub mod wasm_event_stream; diff --git a/mm2src/mm2_net/src/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs similarity index 89% rename from mm2src/mm2_net/src/sse_handler.rs rename to mm2src/mm2_net/src/event_streaming/sse_handler.rs index 1b931142ef..763dd6ce4b 100644 --- a/mm2src/mm2_net/src/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -13,11 +13,11 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result ctx, Err(err) => return handle_internal_error(err).await, }; + let event_streaming_config = ctx.event_streaming_configuration(); + if event_streaming_config.disabled { + return handle_internal_error("Event streaming is explicitly disabled".to_string()).await; + } - let access_control = ctx.conf["access_control_allow_origin"] - .as_str() - .unwrap_or("*") - .to_string(); let Some(Ok(client_id)) = request.uri().query().and_then(|query| { query .split('&') @@ -56,7 +56,10 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Sun, 6 Oct 2024 13:25:15 +0200 Subject: [PATCH 104/128] setting a default to client_id this sets a default for the client id from SSE side (to zero) and from rpc enablers (tasks coming soon) as well. also a new composition struct is added to the rpc enablers to stop the repitition of client_id in each streaming enabler rpc. same technique should be followed with the task manager rpcs --- .../src/rpc/streaming_activations/balance.rs | 30 ++++++------------- .../streaming_activations/fee_estimation.rs | 8 ++--- .../rpc/streaming_activations/heartbeat.rs | 8 ++--- .../src/rpc/streaming_activations/mod.rs | 12 +++++++- .../src/rpc/streaming_activations/network.rs | 8 ++--- .../src/rpc/streaming_activations/orders.rs | 9 ++---- .../src/rpc/streaming_activations/swaps.rs | 9 ++---- .../rpc/streaming_activations/tx_history.rs | 26 +++++----------- .../src/event_streaming/sse_handler.rs | 14 ++++----- 9 files changed, 51 insertions(+), 73 deletions(-) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 5f34539f92..526c4baf5b 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different balance event streamers. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use coins::eth::eth_balance_events::EthBalanceEventStreamer; use coins::tendermint::tendermint_balance_events::TendermintBalanceEventStreamer; @@ -15,7 +15,6 @@ use serde_json::Value as Json; #[derive(Deserialize)] pub struct EnableBalanceStreamingRequest { - pub client_id: u64, pub coin: String, pub config: Option, } @@ -42,8 +41,9 @@ impl HttpStatusCode for BalanceStreamingRequestError { pub async fn enable_balance( ctx: MmArc, - req: EnableBalanceStreamingRequest, + req: EnableStreamingRequest, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind(&ctx, &req.coin) .await .map_err(BalanceStreamingRequestError::Internal)? @@ -53,44 +53,32 @@ pub async fn enable_balance( MmCoinEnum::UtxoCoin(coin) => { check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Bch(coin) => { check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::QtumCoin(coin) => { check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::EthCoin(coin) => { let streamer = EthBalanceEventStreamer::try_new(req.config, coin.clone()) .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}")))?; - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::ZCoin(coin) => { check_empty_config(&req.config)?; let streamer = ZCoinBalanceEventStreamer::new(coin.clone()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Tendermint(coin) => { check_empty_config(&req.config)?; let streamer = TendermintBalanceEventStreamer::new(coin.clone()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, // FIXME: What about tokens?! _ => Err(BalanceStreamingRequestError::CoinNotSupported)?, diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs index 6b80f4040d..ef36542716 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/fee_estimation.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for different fee estimation streamers. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use coins::eth::fee_estimation::eth_fee_events::{EthFeeEventStreamer, EthFeeStreamingConfig}; use coins::{lp_coinfind, MmCoin, MmCoinEnum}; @@ -10,7 +10,6 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; #[derive(Deserialize)] pub struct EnableFeeStreamingRequest { - pub client_id: u64, pub coin: String, pub config: EthFeeStreamingConfig, } @@ -37,8 +36,9 @@ impl HttpStatusCode for FeeStreamingRequestError { pub async fn enable_fee_estimation( ctx: MmArc, - req: EnableFeeStreamingRequest, + req: EnableStreamingRequest, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind(&ctx, &req.coin) .await .map_err(FeeStreamingRequestError::Internal)? @@ -48,7 +48,7 @@ pub async fn enable_fee_estimation( MmCoinEnum::EthCoin(coin) => { let eth_fee_estimator_streamer = EthFeeEventStreamer::new(req.config, coin.clone()); ctx.event_stream_manager - .add(req.client_id, eth_fee_estimator_streamer, coin.spawner()) + .add(client_id, eth_fee_estimator_streamer, coin.spawner()) .await .map(EnableStreamingResponse::new) .map_to_mm(|e| FeeStreamingRequestError::EnableError(format!("{e:?}"))) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs index 2169c3dcaa..e3f4d06c5e 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/heartbeat.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the heartbeats. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use crate::heartbeat_event::{HeartbeatEvent, HeartbeatEventConfig}; use common::HttpStatusCode; @@ -9,7 +9,6 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; #[derive(Deserialize)] pub struct EnableHeartbeatRequest { - pub client_id: u64, pub config: HeartbeatEventConfig, } @@ -25,11 +24,12 @@ impl HttpStatusCode for HeartbeatRequestError { pub async fn enable_heartbeat( ctx: MmArc, - req: EnableHeartbeatRequest, + req: EnableStreamingRequest, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let heartbeat_streamer = HeartbeatEvent::new(req.config); ctx.event_stream_manager - .add(req.client_id, heartbeat_streamer, ctx.spawner()) + .add(client_id, heartbeat_streamer, ctx.spawner()) .await .map(EnableStreamingResponse::new) .map_to_mm(|e| HeartbeatRequestError::EnableError(format!("{e:?}"))) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index c06311abaa..5155b13eb9 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -17,9 +17,19 @@ pub use orders::*; pub use swaps::*; pub use tx_history::*; +#[derive(Deserialize)] +/// The general request for enabling any streamer. +/// `client_id` is common in each request, other data is request-specific. +pub struct EnableStreamingRequest { + // If the client ID isn't included, assume it's 0. + #[serde(default)] + pub client_id: u64, + #[serde(flatten)] + inner: T, +} + #[derive(Serialize)] /// The success/ok response for any event streaming activation request. -/// Note that we don't have a unified request. It is rather defined per event streaming activation. pub struct EnableStreamingResponse { pub streamer_id: String, // FIXME: If the the streamer was already running, it is probably running with different configuration. diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs index 4310476828..bebfdff131 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/network.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/network.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for the network event streamer. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use common::HttpStatusCode; use http::StatusCode; @@ -9,7 +9,6 @@ use mm2_net::network_event::{NetworkEvent, NetworkEventConfig}; #[derive(Deserialize)] pub struct EnableNetworkStreamingRequest { - pub client_id: u64, pub config: NetworkEventConfig, } @@ -25,11 +24,12 @@ impl HttpStatusCode for NetworkStreamingRequestError { pub async fn enable_network( ctx: MmArc, - req: EnableNetworkStreamingRequest, + req: EnableStreamingRequest, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let network_steamer = NetworkEvent::new(req.config, ctx.clone()); ctx.event_stream_manager - .add(req.client_id, network_steamer, ctx.spawner()) + .add(client_id, network_steamer, ctx.spawner()) .await .map(EnableStreamingResponse::new) .map_to_mm(|e| NetworkStreamingRequestError::EnableError(format!("{e:?}"))) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs index 8715cdc51d..08fd0a959a 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orders.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation of the order status streamer. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use crate::lp_ordermatch::order_events::OrderStatusStreamer; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; @@ -7,11 +7,6 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; use common::HttpStatusCode; use http::StatusCode; -#[derive(Deserialize)] -pub struct EnableOrderStatusStreamingRequest { - pub client_id: u64, -} - #[derive(Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum OrderStatusStreamingRequestError { @@ -24,7 +19,7 @@ impl HttpStatusCode for OrderStatusStreamingRequestError { pub async fn enable_order_status( ctx: MmArc, - req: EnableOrderStatusStreamingRequest, + req: EnableStreamingRequest<()>, ) -> MmResult { let order_status_streamer = OrderStatusStreamer::new(); ctx.event_stream_manager diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs index d4a8f59251..3c3cb5d248 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation of the swap status streamer. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use crate::lp_swap::swap_events::SwapStatusStreamer; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; @@ -7,11 +7,6 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; use common::HttpStatusCode; use http::StatusCode; -#[derive(Deserialize)] -pub struct EnableSwapStatusStreamingRequest { - pub client_id: u64, -} - #[derive(Display, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum SwapStatusStreamingRequestError { @@ -24,7 +19,7 @@ impl HttpStatusCode for SwapStatusStreamingRequestError { pub async fn enable_swap_status( ctx: MmArc, - req: EnableSwapStatusStreamingRequest, + req: EnableStreamingRequest<()>, ) -> MmResult { let swap_status_streamer = SwapStatusStreamer::new(); ctx.event_stream_manager diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs index cafa69d686..4b0775fe7e 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs @@ -1,5 +1,5 @@ //! RPC activation and deactivation for Tx history event streamers. -use super::EnableStreamingResponse; +use super::{EnableStreamingRequest, EnableStreamingResponse}; use coins::utxo::tx_history_events::TxHistoryEventStreamer; use coins::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; @@ -11,7 +11,6 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; #[derive(Deserialize)] pub struct EnableTxHistoryStreamingRequest { - pub client_id: u64, pub coin: String, } @@ -37,8 +36,9 @@ impl HttpStatusCode for TxHistoryStreamingRequestError { pub async fn enable_tx_history( ctx: MmArc, - req: EnableTxHistoryStreamingRequest, + req: EnableStreamingRequest, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind(&ctx, &req.coin) .await .map_err(TxHistoryStreamingRequestError::Internal)? @@ -47,35 +47,25 @@ pub async fn enable_tx_history( let enable_result = match coin { MmCoinEnum::UtxoCoin(coin) => { let streamer = TxHistoryEventStreamer::new(req.coin); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Bch(coin) => { let streamer = TxHistoryEventStreamer::new(req.coin); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::QtumCoin(coin) => { let streamer = TxHistoryEventStreamer::new(req.coin); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Tendermint(coin) => { // The tx history streamer is very primitive reactive streamer that only emits new txs. // it's logic is exactly the same for utxo coins and tendermint coins as well. let streamer = TxHistoryEventStreamer::new(req.coin); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::ZCoin(coin) => { let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); - ctx.event_stream_manager - .add(req.client_id, streamer, coin.spawner()) - .await + ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, // FIXME: What about tokens?! _ => Err(TxHistoryStreamingRequestError::CoinNotSupported)?, diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index 763dd6ce4b..ec4906d515 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -7,25 +7,25 @@ pub const SSE_ENDPOINT: &str = "/event-stream"; /// Handles broadcasted messages from `mm2_event_stream` continuously. pub async fn handle_sse(request: Request, ctx_h: u32) -> Result, Infallible> { - // This is only called once for per client on the initialization, - // meaning this is not a resource intensive computation. let ctx = match MmArc::from_ffi_handle(ctx_h) { Ok(ctx) => ctx, Err(err) => return handle_internal_error(err).await, }; + let event_streaming_config = ctx.event_streaming_configuration(); if event_streaming_config.disabled { return handle_internal_error("Event streaming is explicitly disabled".to_string()).await; } - let Some(Ok(client_id)) = request.uri().query().and_then(|query| { + let client_id = match request.uri().query().and_then(|query| { query .split('&') .find(|param| param.starts_with("id=")) - .map(|id_param| id_param.trim_start_matches("id=")) - .map(|id| id.parse::()) - }) else { - return handle_internal_error("Query parameter `id` (64-bit unsigned number) must be present.".to_string()).await; + .map(|id_param| id_param.trim_start_matches("id=").parse::()) + }) { + Some(Ok(id)) => id, + // Default to zero when client ID isn't passed, most of the cases we will have a single user/client. + _ => 0, }; let event_stream_manager = ctx.event_stream_manager.clone(); From 91d98edbefc3edd1b68c8bbf912c2db31a5718ed Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sun, 6 Oct 2024 14:52:23 +0200 Subject: [PATCH 105/128] set default cliend_id for rpc tasks this sets a default client id to zero for rpc managed tasks, so for the default case of single user they don't have to provide an ID to get progress updates. this also introduces a new composition struct `RpcInitReq` for init rpcs which that holds the clinet_id instead of having to declare it in each RPC request. also the client_id is not removed from the RpcTask and supplied to the task manager directly so we don't have to store the clinet_id in each task as well. --- mm2src/coins/lp_coins.rs | 24 ++++++++++--------- mm2src/coins/rpc_command/get_new_address.rs | 13 +++++----- .../coins/rpc_command/init_account_balance.rs | 12 +++++----- .../coins/rpc_command/init_create_account.rs | 13 +++++----- .../init_scan_for_new_addresses.rs | 12 +++++----- mm2src/coins/rpc_command/init_withdraw.rs | 13 ++++++---- mm2src/coins_activation/src/init_token.rs | 12 ++++------ mm2src/coins_activation/src/l2/init_l2.rs | 14 ++++------- .../src/platform_coin_with_tokens.rs | 18 +++++++------- .../standalone_coin/init_standalone_coin.rs | 11 ++++----- .../src/utxo_activation/mod.rs | 6 ++++- mm2src/mm2_main/src/lp_init/init_hw.rs | 12 ++++------ mm2src/mm2_main/src/lp_init/init_metamask.rs | 12 ++++------ .../tests/mm2_tests/mm2_tests_inner.rs | 4 ++-- mm2src/rpc_task/src/lib.rs | 2 +- mm2src/rpc_task/src/manager.rs | 19 +++++++++------ mm2src/rpc_task/src/task.rs | 18 ++++++++++---- 17 files changed, 113 insertions(+), 102 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 01302a38ba..a9badd82b6 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -2177,7 +2177,6 @@ pub struct WithdrawRequest { #[cfg(target_arch = "wasm32")] #[serde(default)] broadcast: bool, - client_id: Option, } #[derive(Debug, Deserialize)] @@ -5666,7 +5665,7 @@ pub mod for_tests { use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmResult; use mm2_number::BigDecimal; - use rpc_task::RpcTaskStatus; + use rpc_task::{RpcInitReq, RpcTaskStatus}; use std::str::FromStr; /// Helper to call init_withdraw and wait for completion @@ -5678,15 +5677,18 @@ pub mod for_tests { from_derivation_path: Option<&str>, fee: Option, ) -> MmResult { - let withdraw_req = WithdrawRequest { - amount: BigDecimal::from_str(amount).unwrap(), - from: from_derivation_path.map(|from_derivation_path| WithdrawFrom::DerivationPath { - derivation_path: from_derivation_path.to_owned(), - }), - to: to.to_owned(), - coin: ticker.to_owned(), - fee, - ..Default::default() + let withdraw_req = RpcInitReq { + client_id: 0, + inner: WithdrawRequest { + amount: BigDecimal::from_str(amount).unwrap(), + from: from_derivation_path.map(|from_derivation_path| WithdrawFrom::DerivationPath { + derivation_path: from_derivation_path.to_owned(), + }), + to: to.to_owned(), + coin: ticker.to_owned(), + fee, + ..Default::default() + }, }; let init = init_withdraw(ctx.clone(), withdraw_req).await.unwrap(); let timeout = wait_until_ms(150000); diff --git a/mm2src/coins/rpc_command/get_new_address.rs b/mm2src/coins/rpc_command/get_new_address.rs index aa3636f918..a4a1a0c5ce 100644 --- a/mm2src/coins/rpc_command/get_new_address.rs +++ b/mm2src/coins/rpc_command/get_new_address.rs @@ -16,8 +16,8 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, - RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, + RpcTaskStatus, RpcTaskTypes}; use std::time::Duration; pub type GetNewAddressUserAction = HwRpcTaskUserAction; @@ -204,7 +204,6 @@ pub struct GetNewAddressRequest { coin: String, #[serde(flatten)] params: GetNewAddressParams, - client_id: Option, } #[derive(Clone, Deserialize)] @@ -292,8 +291,6 @@ impl RpcTaskTypes for InitGetNewAddressTask { impl RpcTask for InitGetNewAddressTask { fn initial_status(&self) -> Self::InProgressStatus { GetNewAddressInProgressStatus::Preparing } - fn client_id(&self) -> Option { self.req.client_id } - // Do nothing if the task has been cancelled. async fn cancel(self) {} @@ -382,13 +379,15 @@ pub async fn get_new_address( /// TODO remove once GUI integrates `task::get_new_address::init`. pub async fn init_get_new_address( ctx: MmArc, - req: GetNewAddressRequest, + req: RpcInitReq, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(GetNewAddressRpcError::Internal)?; let spawner = coin.spawner(); let task = InitGetNewAddressTask { ctx, coin, req }; - let task_id = GetNewAddressTaskManager::spawn_rpc_task(&coins_ctx.get_new_address_manager, &spawner, task)?; + let task_id = + GetNewAddressTaskManager::spawn_rpc_task(&coins_ctx.get_new_address_manager, &spawner, task, client_id)?; Ok(InitRpcTaskResponse { task_id }) } diff --git a/mm2src/coins/rpc_command/init_account_balance.rs b/mm2src/coins/rpc_command/init_account_balance.rs index e7d7cea183..287c40dcb8 100644 --- a/mm2src/coins/rpc_command/init_account_balance.rs +++ b/mm2src/coins/rpc_command/init_account_balance.rs @@ -7,7 +7,8 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest}; -use rpc_task::{RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, + RpcTaskTypes}; pub type AccountBalanceUserAction = SerdeInfallible; pub type AccountBalanceAwaitingStatus = SerdeInfallible; @@ -31,7 +32,6 @@ pub struct InitAccountBalanceRequest { coin: String, #[serde(flatten)] params: InitAccountBalanceParams, - client_id: Option, } #[derive(Clone, Deserialize)] @@ -66,8 +66,6 @@ impl RpcTaskTypes for InitAccountBalanceTask { impl RpcTask for InitAccountBalanceTask { fn initial_status(&self) -> Self::InProgressStatus { AccountBalanceInProgressStatus::RequestingAccountBalance } - fn client_id(&self) -> Option { self.req.client_id } - // Do nothing if the task has been cancelled. async fn cancel(self) {} @@ -92,13 +90,15 @@ impl RpcTask for InitAccountBalanceTask { pub async fn init_account_balance( ctx: MmArc, - req: InitAccountBalanceRequest, + req: RpcInitReq, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let spawner = coin.spawner(); let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(HDAccountBalanceRpcError::Internal)?; let task = InitAccountBalanceTask { coin, req }; - let task_id = AccountBalanceTaskManager::spawn_rpc_task(&coins_ctx.account_balance_task_manager, &spawner, task)?; + let task_id = + AccountBalanceTaskManager::spawn_rpc_task(&coins_ctx.account_balance_task_manager, &spawner, task, client_id)?; Ok(InitRpcTaskResponse { task_id }) } diff --git a/mm2src/coins/rpc_command/init_create_account.rs b/mm2src/coins/rpc_command/init_create_account.rs index 695b3e5170..ae139f09a5 100644 --- a/mm2src/coins/rpc_command/init_create_account.rs +++ b/mm2src/coins/rpc_command/init_create_account.rs @@ -14,8 +14,8 @@ use mm2_err_handle::prelude::*; use parking_lot::Mutex as PaMutex; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, - RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, + RpcTaskStatus, RpcTaskTypes}; use std::sync::Arc; use std::time::Duration; @@ -164,7 +164,6 @@ pub struct CreateNewAccountRequest { coin: String, #[serde(flatten)] params: CreateNewAccountParams, - client_id: Option, } #[derive(Clone, Deserialize)] @@ -239,8 +238,6 @@ impl RpcTaskTypes for InitCreateAccountTask { impl RpcTask for InitCreateAccountTask { fn initial_status(&self) -> Self::InProgressStatus { CreateAccountInProgressStatus::Preparing } - fn client_id(&self) -> Option { self.req.client_id } - async fn cancel(self) { if let Some(account_id) = self.task_state.create_account_id() { // We created the account already, so need to revert the changes. @@ -332,8 +329,9 @@ impl RpcTask for InitCreateAccountTask { pub async fn init_create_new_account( ctx: MmArc, - req: CreateNewAccountRequest, + req: RpcInitReq, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(CreateAccountRpcError::Internal)?; let spawner = coin.spawner(); @@ -343,7 +341,8 @@ pub async fn init_create_new_account( req, task_state: CreateAccountState::default(), }; - let task_id = CreateAccountTaskManager::spawn_rpc_task(&coins_ctx.create_account_manager, &spawner, task)?; + let task_id = + CreateAccountTaskManager::spawn_rpc_task(&coins_ctx.create_account_manager, &spawner, task, client_id)?; Ok(InitRpcTaskResponse { task_id }) } diff --git a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs index ebbb4d4b86..5195248b7d 100644 --- a/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs +++ b/mm2src/coins/rpc_command/init_scan_for_new_addresses.rs @@ -8,7 +8,8 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest}; -use rpc_task::{RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, + RpcTaskTypes}; pub type ScanAddressesUserAction = SerdeInfallible; pub type ScanAddressesAwaitingStatus = SerdeInfallible; @@ -43,7 +44,6 @@ pub struct ScanAddressesRequest { coin: String, #[serde(flatten)] params: ScanAddressesParams, - client_id: Option, } #[derive(Clone, Deserialize)] @@ -88,8 +88,6 @@ impl RpcTask for InitScanAddressesTask { #[inline] fn initial_status(&self) -> Self::InProgressStatus { ScanAddressesInProgressStatus::InProgress } - fn client_id(&self) -> Option { self.req.client_id } - // Do nothing if the task has been cancelled. async fn cancel(self) {} @@ -111,13 +109,15 @@ impl RpcTask for InitScanAddressesTask { pub async fn init_scan_for_new_addresses( ctx: MmArc, - req: ScanAddressesRequest, + req: RpcInitReq, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let coin = lp_coinfind_or_err(&ctx, &req.coin).await?; let spawner = coin.spawner(); let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(HDAccountBalanceRpcError::Internal)?; let task = InitScanAddressesTask { req, coin }; - let task_id = ScanAddressesTaskManager::spawn_rpc_task(&coins_ctx.scan_addresses_manager, &spawner, task)?; + let task_id = + ScanAddressesTaskManager::spawn_rpc_task(&coins_ctx.scan_addresses_manager, &spawner, task, client_id)?; Ok(InitRpcTaskResponse { task_id }) } diff --git a/mm2src/coins/rpc_command/init_withdraw.rs b/mm2src/coins/rpc_command/init_withdraw.rs index 09728d4785..e82ccd4d63 100644 --- a/mm2src/coins/rpc_command/init_withdraw.rs +++ b/mm2src/coins/rpc_command/init_withdraw.rs @@ -7,7 +7,8 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError}; -use rpc_task::{RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatusAlias, RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatusAlias, + RpcTaskTypes}; pub type WithdrawAwaitingStatus = HwRpcTaskAwaitingStatus; pub type WithdrawUserAction = HwRpcTaskUserAction; @@ -32,7 +33,11 @@ pub trait CoinWithdrawInit { ) -> WithdrawInitResult; } -pub async fn init_withdraw(ctx: MmArc, request: WithdrawRequest) -> WithdrawInitResult { +pub async fn init_withdraw( + ctx: MmArc, + request: RpcInitReq, +) -> WithdrawInitResult { + let (client_id, request) = (request.client_id, request.inner); let coin = lp_coinfind_or_err(&ctx, &request.coin).await?; let spawner = coin.spawner(); let task = WithdrawTask { @@ -41,7 +46,7 @@ pub async fn init_withdraw(ctx: MmArc, request: WithdrawRequest) -> WithdrawInit request, }; let coins_ctx = CoinsContext::from_ctx(&ctx).map_to_mm(WithdrawError::InternalError)?; - let task_id = WithdrawTaskManager::spawn_rpc_task(&coins_ctx.withdraw_task_manager, &spawner, task)?; + let task_id = WithdrawTaskManager::spawn_rpc_task(&coins_ctx.withdraw_task_manager, &spawner, task, client_id)?; Ok(InitWithdrawResponse { task_id }) } @@ -123,8 +128,6 @@ impl RpcTaskTypes for WithdrawTask { impl RpcTask for WithdrawTask { fn initial_status(&self) -> Self::InProgressStatus { WithdrawInProgressStatus::Preparing } - fn client_id(&self) -> Option { self.request.client_id } - // Do nothing if the task has been cancelled. async fn cancel(self) {} diff --git a/mm2src/coins_activation/src/init_token.rs b/mm2src/coins_activation/src/init_token.rs index 38e433285f..58d9ed9180 100644 --- a/mm2src/coins_activation/src/init_token.rs +++ b/mm2src/coins_activation/src/init_token.rs @@ -15,8 +15,8 @@ use mm2_err_handle::mm_error::{MmError, MmResult, NotEqual, NotMmError}; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError, RpcTaskUserActionRequest}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, - RpcTaskTypes, TaskId}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, + RpcTaskStatus, RpcTaskTypes, TaskId}; use ser_error_derive::SerializeErrorType; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; @@ -38,7 +38,6 @@ pub type CancelInitTokenError = CancelRpcTaskError; pub struct InitTokenReq { ticker: String, activation_params: T, - client_id: Option, } /// Trait for the initializing a token using the task manager. @@ -83,7 +82,7 @@ pub trait InitTokenActivationOps: Into + TokenOf + Clone + Send + Sy /// Implementation of the init token RPC command. pub async fn init_token( ctx: MmArc, - request: InitTokenReq, + request: RpcInitReq>, ) -> MmResult where Token: InitTokenActivationOps + Send + Sync + 'static, @@ -91,6 +90,7 @@ where InitTokenError: From, (Token::ActivationError, InitTokenError): NotEqual, { + let (client_id, request) = (request.client_id, request.inner); if let Ok(Some(_)) = lp_coinfind(&ctx, &request.ticker).await { return MmError::err(InitTokenError::TokenIsAlreadyActivated { ticker: request.ticker }); } @@ -117,7 +117,7 @@ where }; let task_manager = Token::rpc_task_manager(&coins_act_ctx); - let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task) + let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task, client_id) .mm_err(|e| InitTokenError::Internal(e.to_string()))?; Ok(InitTokenResponse { task_id }) @@ -196,8 +196,6 @@ where ::initial_status() } - fn client_id(&self) -> Option { self.request.client_id } - /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(c_ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/coins_activation/src/l2/init_l2.rs b/mm2src/coins_activation/src/l2/init_l2.rs index ab59fc2a6c..14aee68afe 100644 --- a/mm2src/coins_activation/src/l2/init_l2.rs +++ b/mm2src/coins_activation/src/l2/init_l2.rs @@ -10,7 +10,8 @@ use common::SuccessResponse; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusRequest, RpcTaskUserActionRequest}; -use rpc_task::{RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, + RpcTaskTypes}; use serde_derive::Deserialize; use serde_json::Value as Json; @@ -24,7 +25,6 @@ pub type InitL2TaskHandleShared = RpcTaskHandleShared>; pub struct InitL2Req { ticker: String, activation_params: T, - client_id: Option, } pub trait L2ProtocolParams { @@ -68,13 +68,14 @@ pub trait InitL2ActivationOps: Into + Send + Sync + 'static { pub async fn init_l2( ctx: MmArc, - req: InitL2Req, + req: RpcInitReq>, ) -> Result> where L2: InitL2ActivationOps, InitL2Error: From, (L2::ActivationError, InitL2Error): NotEqual, { + let (client_id, req) = (req.client_id, req.inner); let ticker = req.ticker.clone(); if let Ok(Some(_)) = lp_coinfind(&ctx, &ticker).await { return MmError::err(InitL2Error::L2IsAlreadyActivated(ticker)); @@ -99,11 +100,9 @@ where let coins_act_ctx = CoinsActivationContext::from_ctx(&ctx).map_to_mm(InitL2Error::Internal)?; let spawner = ctx.spawner(); - let client_id = req.client_id; let task = InitL2Task:: { ctx, ticker, - client_id, platform_coin, validated_params, protocol_conf, @@ -111,7 +110,7 @@ where }; let task_manager = L2::rpc_task_manager(&coins_act_ctx); - let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task) + let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task, client_id) .mm_err(|e| InitL2Error::Internal(e.to_string()))?; Ok(InitL2Response { task_id }) @@ -164,7 +163,6 @@ pub async fn cancel_init_l2( pub struct InitL2Task { ctx: MmArc, ticker: String, - client_id: Option, platform_coin: L2::PlatformCoin, validated_params: L2::ValidatedParams, protocol_conf: L2::ProtocolInfo, @@ -188,8 +186,6 @@ where ::initial_status() } - fn client_id(&self) -> Option { self.client_id } - /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/coins_activation/src/platform_coin_with_tokens.rs b/mm2src/coins_activation/src/platform_coin_with_tokens.rs index 0c0c1f6106..ed72183116 100644 --- a/mm2src/coins_activation/src/platform_coin_with_tokens.rs +++ b/mm2src/coins_activation/src/platform_coin_with_tokens.rs @@ -16,8 +16,8 @@ use mm2_err_handle::prelude::*; use mm2_number::BigDecimal; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError, RpcTaskUserActionRequest}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, - RpcTaskTypes, TaskId}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, + RpcTaskStatus, RpcTaskTypes, TaskId}; use ser_error_derive::SerializeErrorType; use serde_derive::{Deserialize, Serialize}; use serde_json::Value as Json; @@ -223,7 +223,6 @@ pub struct EnablePlatformCoinWithTokensReq { ticker: String, #[serde(flatten)] request: T, - client_id: Option, } #[derive(Debug, Display, Serialize, SerializeErrorType, Clone)] @@ -510,8 +509,6 @@ where ::initial_status() } - fn client_id(&self) -> Option { self.request.client_id } - /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) {} @@ -550,7 +547,7 @@ impl InitPlatformCoinWithTokensInitialStatus for InitPlatformCoinWithTokensInPro /// Implementation of the init platform coin with tokens RPC command. pub async fn init_platform_coin_with_tokens( ctx: MmArc, - request: EnablePlatformCoinWithTokensReq, + request: RpcInitReq>, ) -> MmResult where Platform: PlatformCoinWithTokensActivationOps + Send + Sync + 'static + Clone, @@ -558,6 +555,7 @@ where EnablePlatformCoinWithTokensError: From, (Platform::ActivationError, EnablePlatformCoinWithTokensError): NotEqual, { + let (client_id, request) = (request.client_id, request.inner); if let Ok(Some(_)) = lp_coinfind(&ctx, &request.ticker).await { return MmError::err(EnablePlatformCoinWithTokensError::PlatformIsAlreadyActivated( request.ticker, @@ -570,7 +568,7 @@ where let task = InitPlatformCoinWithTokensTask:: { ctx, request }; let task_manager = Platform::rpc_task_manager(&coins_act_ctx); - let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task) + let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task, client_id) .mm_err(|e| EnablePlatformCoinWithTokensError::Internal(e.to_string()))?; Ok(EnablePlatformCoinWithTokensResponse { task_id }) @@ -654,7 +652,7 @@ pub mod for_tests { use common::{executor::Timer, now_ms, wait_until_ms}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmResult; - use rpc_task::RpcTaskStatus; + use rpc_task::{RpcInitReq, RpcTaskStatus}; use super::{init_platform_coin_with_tokens, init_platform_coin_with_tokens_status, EnablePlatformCoinWithTokensError, EnablePlatformCoinWithTokensReq, @@ -672,6 +670,10 @@ pub mod for_tests { EnablePlatformCoinWithTokensError: From, (Platform::ActivationError, EnablePlatformCoinWithTokensError): NotEqual, { + let request = RpcInitReq { + client_id: 0, + inner: request, + }; let init_result = init_platform_coin_with_tokens::(ctx.clone(), request) .await .unwrap(); diff --git a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs index 3288832add..c4e7fa8c92 100644 --- a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs +++ b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs @@ -13,7 +13,8 @@ use mm2_err_handle::prelude::*; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use rpc_task::rpc_common::{CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusRequest, RpcTaskUserActionRequest}; -use rpc_task::{RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, + RpcTaskTypes}; use serde_derive::Deserialize; use serde_json::Value as Json; use std::collections::HashMap; @@ -28,7 +29,6 @@ pub type InitStandaloneCoinTaskHandleShared = RpcTaskHandleShared { ticker: String, activation_params: T, - client_id: Option, } #[async_trait] @@ -79,7 +79,7 @@ pub trait InitStandaloneCoinActivationOps: Into + Send + Sync + 'sta pub async fn init_standalone_coin( ctx: MmArc, - request: InitStandaloneCoinReq, + request: RpcInitReq>, ) -> MmResult where Standalone: InitStandaloneCoinActivationOps + Send + Sync + 'static, @@ -87,6 +87,7 @@ where InitStandaloneCoinError: From, (Standalone::ActivationError, InitStandaloneCoinError): NotEqual, { + let (client_id, request) = (request.client_id, request.inner); if let Ok(Some(_)) = lp_coinfind(&ctx, &request.ticker).await { return MmError::err(InitStandaloneCoinError::CoinIsAlreadyActivated { ticker: request.ticker }); } @@ -103,7 +104,7 @@ where }; let task_manager = Standalone::rpc_task_manager(&coins_act_ctx); - let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task) + let task_id = RpcTaskManager::spawn_rpc_task(task_manager, &spawner, task, client_id) .mm_err(|e| InitStandaloneCoinError::Internal(e.to_string()))?; Ok(InitStandaloneCoinResponse { task_id }) @@ -184,8 +185,6 @@ where ::initial_status() } - fn client_id(&self) -> Option { self.request.client_id } - /// Try to disable the coin in case if we managed to register it already. async fn cancel(self) { if let Ok(c_ctx) = CoinsContext::from_ctx(&self.ctx) { diff --git a/mm2src/coins_activation/src/utxo_activation/mod.rs b/mm2src/coins_activation/src/utxo_activation/mod.rs index 52244ed2dc..42764e5c93 100644 --- a/mm2src/coins_activation/src/utxo_activation/mod.rs +++ b/mm2src/coins_activation/src/utxo_activation/mod.rs @@ -16,7 +16,7 @@ pub mod for_tests { use common::{executor::Timer, now_ms, wait_until_ms}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::{MmResult, NotEqual}; - use rpc_task::RpcTaskStatus; + use rpc_task::{RpcInitReq, RpcTaskStatus}; use crate::{init_standalone_coin, init_standalone_coin_status, standalone_coin::{InitStandaloneCoinActivationOps, InitStandaloneCoinError, @@ -34,6 +34,10 @@ pub mod for_tests { InitStandaloneCoinError: From, (Standalone::ActivationError, InitStandaloneCoinError): NotEqual, { + let request = RpcInitReq { + client_id: 0, + inner: request, + }; let init_result = init_standalone_coin::(ctx.clone(), request).await.unwrap(); let timeout = wait_until_ms(150000); loop { diff --git a/mm2src/mm2_main/src/lp_init/init_hw.rs b/mm2src/mm2_main/src/lp_init/init_hw.rs index 0315b17aa3..6148a44f53 100644 --- a/mm2src/mm2_main/src/lp_init/init_hw.rs +++ b/mm2src/mm2_main/src/lp_init/init_hw.rs @@ -12,8 +12,8 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use rpc_task::rpc_common::{CancelRpcTaskError, CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusError, RpcTaskStatusRequest, RpcTaskUserActionError}; -use rpc_task::{RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, RpcTaskStatus, - RpcTaskTypes}; +use rpc_task::{RpcInitReq, RpcTask, RpcTaskError, RpcTaskHandleShared, RpcTaskManager, RpcTaskManagerShared, + RpcTaskStatus, RpcTaskTypes}; use std::sync::Arc; use std::time::Duration; @@ -101,7 +101,6 @@ pub enum InitHwInProgressStatus { #[derive(Deserialize, Clone)] pub struct InitHwRequest { device_pubkey: Option, - client_id: Option, } #[derive(Clone, Serialize, Debug, Deserialize)] @@ -130,8 +129,6 @@ impl RpcTaskTypes for InitHwTask { impl RpcTask for InitHwTask { fn initial_status(&self) -> Self::InProgressStatus { InitHwInProgressStatus::Initializing } - fn client_id(&self) -> Option { self.req.client_id } - async fn cancel(self) { if let Ok(crypto_ctx) = CryptoCtx::from_ctx(&self.ctx) { crypto_ctx.reset_hw_ctx() @@ -168,7 +165,8 @@ impl RpcTask for InitHwTask { } } -pub async fn init_trezor(ctx: MmArc, req: InitHwRequest) -> MmResult { +pub async fn init_trezor(ctx: MmArc, req: RpcInitReq) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let init_ctx = MmInitContext::from_ctx(&ctx).map_to_mm(InitHwError::Internal)?; let spawner = ctx.spawner(); let task = InitHwTask { @@ -176,7 +174,7 @@ pub async fn init_trezor(ctx: MmArc, req: InitHwRequest) -> MmResult, } #[derive(Clone, Serialize)] @@ -116,8 +115,6 @@ impl RpcTaskTypes for InitMetamaskTask { impl RpcTask for InitMetamaskTask { fn initial_status(&self) -> Self::InProgressStatus { InitMetamaskInProgressStatus::Initializing } - fn client_id(&self) -> Option { self.req.client_id } - async fn cancel(self) { if let Ok(crypto_ctx) = CryptoCtx::from_ctx(&self.ctx) { crypto_ctx.reset_metamask_ctx(); @@ -136,12 +133,13 @@ impl RpcTask for InitMetamaskTask { pub async fn connect_metamask( ctx: MmArc, - req: InitMetamaskRequest, + req: RpcInitReq, ) -> MmResult { + let (client_id, req) = (req.client_id, req.inner); let init_ctx = MmInitContext::from_ctx(&ctx).map_to_mm(InitMetamaskError::Internal)?; let spawner = ctx.spawner(); let task = InitMetamaskTask { ctx, req }; - let task_id = RpcTaskManager::spawn_rpc_task(&init_ctx.init_metamask_manager, &spawner, task)?; + let task_id = RpcTaskManager::spawn_rpc_task(&init_ctx.init_metamask_manager, &spawner, task, client_id)?; Ok(InitRpcTaskResponse { task_id }) } diff --git a/mm2src/mm2_main/tests/mm2_tests/mm2_tests_inner.rs b/mm2src/mm2_main/tests/mm2_tests/mm2_tests_inner.rs index 57a4709758..a6e312ff3d 100644 --- a/mm2src/mm2_main/tests/mm2_tests/mm2_tests_inner.rs +++ b/mm2src/mm2_main/tests/mm2_tests/mm2_tests_inner.rs @@ -5991,7 +5991,7 @@ mod trezor_tests { withdraw_status, MarketMakerIt, Mm2TestConf, ETH_SEPOLIA_NODES, ETH_SEPOLIA_SWAP_CONTRACT}; use mm2_test_helpers::structs::{InitTaskResult, RpcV2Response, TransactionDetails, WithdrawStatus}; - use rpc_task::{rpc_common::RpcTaskStatusRequest, RpcTaskStatus}; + use rpc_task::{rpc_common::RpcTaskStatusRequest, RpcInitReq, RpcTaskStatus}; use serde_json::{self as json, json, Value as Json}; use std::io::{stdin, stdout, BufRead, Write}; @@ -6008,7 +6008,7 @@ mod trezor_tests { let ctx = mm_ctx_with_custom_db_with_conf(Some(conf)); CryptoCtx::init_with_iguana_passphrase(ctx.clone(), "123456").unwrap(); // for now we need passphrase seed for init - let req: InitHwRequest = serde_json::from_value(json!({ "device_pubkey": null })).unwrap(); + let req: RpcInitReq = serde_json::from_value(json!({ "device_pubkey": null })).unwrap(); let res = match init_trezor(ctx.clone(), req).await { Ok(res) => res, _ => { diff --git a/mm2src/rpc_task/src/lib.rs b/mm2src/rpc_task/src/lib.rs index f5861f37cc..2e8f703d87 100644 --- a/mm2src/rpc_task/src/lib.rs +++ b/mm2src/rpc_task/src/lib.rs @@ -16,7 +16,7 @@ mod task; pub use handle::{RpcTaskHandle, RpcTaskHandleShared}; pub use manager::{RpcTaskManager, RpcTaskManagerShared}; -pub use task::{RpcTask, RpcTaskTypes}; +pub use task::{RpcInitReq, RpcTask, RpcTaskTypes}; pub type RpcTaskResult = Result>; pub type TaskId = u64; diff --git a/mm2src/rpc_task/src/manager.rs b/mm2src/rpc_task/src/manager.rs index 12f1a4ed23..7e8e38fc24 100644 --- a/mm2src/rpc_task/src/manager.rs +++ b/mm2src/rpc_task/src/manager.rs @@ -39,7 +39,12 @@ pub struct RpcTaskManager { impl RpcTaskManager { /// Create new instance of `RpcTaskHandle` attached to the only one `RpcTask`. /// This function registers corresponding RPC task in the `RpcTaskManager` and returns the task id. - pub fn spawn_rpc_task(this: &RpcTaskManagerShared, spawner: &F, mut task: Task) -> RpcTaskResult + pub fn spawn_rpc_task( + this: &RpcTaskManagerShared, + spawner: &F, + mut task: Task, + client_id: u64, + ) -> RpcTaskResult where F: SpawnFuture, { @@ -47,7 +52,7 @@ impl RpcTaskManager { let mut task_manager = this .lock() .map_to_mm(|e| RpcTaskError::Internal(format!("RpcTaskManager is not available: {}", e)))?; - task_manager.register_task(&task)? + task_manager.register_task(&task, client_id)? }; let task_handle = Arc::new(RpcTaskHandle { task_manager: RpcTaskManagerShared::downgrade(this), @@ -117,7 +122,7 @@ impl RpcTaskManager { fn get_client_id(&self, task_id: TaskId) -> Option { self.tasks.get(&task_id).and_then(|task| match task { - TaskStatusExt::InProgress { client_id, .. } | TaskStatusExt::Awaiting { client_id, .. } => *client_id, + TaskStatusExt::InProgress { client_id, .. } | TaskStatusExt::Awaiting { client_id, .. } => Some(*client_id), _ => None, }) } @@ -153,7 +158,7 @@ impl RpcTaskManager { } } - pub(crate) fn register_task(&mut self, task: &Task) -> RpcTaskResult<(TaskId, TaskAbortHandler)> { + pub(crate) fn register_task(&mut self, task: &Task, client_id: u64) -> RpcTaskResult<(TaskId, TaskAbortHandler)> { let task_id = next_rpc_task_id(); let (abort_handle, abort_handler) = oneshot::channel(); match self.tasks.entry(task_id) { @@ -162,7 +167,7 @@ impl RpcTaskManager { entry.insert(TaskStatusExt::InProgress { status: task.initial_status(), abort_handle, - client_id: task.client_id(), + client_id, }); Ok((task_id, abort_handler)) }, @@ -347,7 +352,7 @@ enum TaskStatusExt { status: Task::InProgressStatus, abort_handle: TaskAbortHandle, /// The ID of the client requesting the task. To stream out the updates & results for them. - client_id: Option, + client_id: u64, }, Awaiting { status: Task::AwaitingStatus, @@ -355,7 +360,7 @@ enum TaskStatusExt { next_in_progress_status: Task::InProgressStatus, abort_handle: TaskAbortHandle, /// The ID of the client requesting the task. To stream out the updates & results for them. - client_id: Option, + client_id: u64, }, /// `Cancelling` status is set on [`RpcTaskManager::cancel_task`]. /// This status is used to save the task state before it's actually canceled on [`RpcTaskHandle::on_canceled`], diff --git a/mm2src/rpc_task/src/task.rs b/mm2src/rpc_task/src/task.rs index c66fb4cef0..06f50e87da 100644 --- a/mm2src/rpc_task/src/task.rs +++ b/mm2src/rpc_task/src/task.rs @@ -15,13 +15,21 @@ pub trait RpcTaskTypes { pub trait RpcTask: RpcTaskTypes + Sized + Send + 'static { fn initial_status(&self) -> Self::InProgressStatus; - /// Returns the ID of the client that initiated/requesting the task. - /// - /// This is related to event streaming and is used to identify the client to whom the updates shall be sent. - fn client_id(&self) -> Option; - /// The method is invoked when the task has been cancelled. async fn cancel(self); async fn run(&mut self, task_handle: RpcTaskHandleShared) -> Result>; } + +#[derive(Deserialize)] +/// The general request for initializing an RPC Task. +/// +/// `client_id` is used to identify the client to which the task should stream out update events +/// to and is common in each request. Other data is request-specific. +pub struct RpcInitReq { + // If the client ID isn't included, assume it's 0. + #[serde(default)] + pub client_id: u64, + #[serde(flatten)] + pub inner: T, +} From 868ee2c596777cb6f652958a1569787cca0da575 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sun, 6 Oct 2024 15:20:29 +0200 Subject: [PATCH 106/128] resolve fixmes --- mm2src/mm2_main/src/rpc/streaming_activations/balance.rs | 1 - mm2src/mm2_main/src/rpc/streaming_activations/mod.rs | 2 +- mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs | 1 - mm2src/mm2_net/src/event_streaming/sse_handler.rs | 6 +----- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index 526c4baf5b..fa49f0a8d6 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -80,7 +80,6 @@ pub async fn enable_balance( let streamer = TendermintBalanceEventStreamer::new(coin.clone()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, - // FIXME: What about tokens?! _ => Err(BalanceStreamingRequestError::CoinNotSupported)?, }; diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index 5155b13eb9..c0a9bdf800 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -32,7 +32,7 @@ pub struct EnableStreamingRequest { /// The success/ok response for any event streaming activation request. pub struct EnableStreamingResponse { pub streamer_id: String, - // FIXME: If the the streamer was already running, it is probably running with different configuration. + // TODO: If the the streamer was already running, it is probably running with different configuration. // We might want to inform the client that the configuration they asked for wasn't applied and return // the active configuration instead? // pub config: Json, diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs index 4b0775fe7e..ac37ca21b5 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/tx_history.rs @@ -67,7 +67,6 @@ pub async fn enable_tx_history( let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, - // FIXME: What about tokens?! _ => Err(TxHistoryStreamingRequestError::CoinNotSupported)?, }; diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index ec4906d515..84eda03cc1 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -30,11 +30,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result Date: Sat, 12 Oct 2024 20:34:21 +0200 Subject: [PATCH 107/128] let event streaming be off by default --- mm2src/mm2_core/src/mm_ctx.rs | 4 ++-- mm2src/mm2_event_stream/src/configuration.rs | 2 -- mm2src/mm2_main/src/lp_native_dex.rs | 3 +-- mm2src/mm2_net/src/event_streaming/sse_handler.rs | 7 +++---- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/mm2src/mm2_core/src/mm_ctx.rs b/mm2src/mm2_core/src/mm_ctx.rs index 8ff812caee..c0f15dc164 100644 --- a/mm2src/mm2_core/src/mm_ctx.rs +++ b/mm2src/mm2_core/src/mm_ctx.rs @@ -333,8 +333,8 @@ impl MmCtx { pub fn use_trading_proto_v2(&self) -> bool { self.conf["use_trading_proto_v2"].as_bool().unwrap_or_default() } /// Returns the event streaming configuration in use. - pub fn event_streaming_configuration(&self) -> EventStreamingConfiguration { - serde_json::from_value(self.conf["event_streaming_configuration"].clone()).unwrap_or_default() + pub fn event_streaming_configuration(&self) -> Option { + serde_json::from_value(self.conf["event_streaming_configuration"].clone()).ok() } /// Returns the cloneable `WeakSpawner`. diff --git a/mm2src/mm2_event_stream/src/configuration.rs b/mm2src/mm2_event_stream/src/configuration.rs index 532d195e76..590665d581 100644 --- a/mm2src/mm2_event_stream/src/configuration.rs +++ b/mm2src/mm2_event_stream/src/configuration.rs @@ -5,7 +5,6 @@ use serde::Deserialize; /// The network-related configuration of the event streaming interface. // TODO: This better fits in mm2_net but then we would have circular dependency error trying to import it in mm2_core. pub struct EventStreamingConfiguration { - pub disabled: bool, pub worker_path: String, pub access_control_allow_origin: String, } @@ -13,7 +12,6 @@ pub struct EventStreamingConfiguration { impl Default for EventStreamingConfiguration { fn default() -> Self { Self { - disabled: false, worker_path: "event_streaming_worker.js".to_string(), access_control_allow_origin: "*".to_string(), } diff --git a/mm2src/mm2_main/src/lp_native_dex.rs b/mm2src/mm2_main/src/lp_native_dex.rs index a23cad1ad7..ff45f2374e 100644 --- a/mm2src/mm2_main/src/lp_native_dex.rs +++ b/mm2src/mm2_main/src/lp_native_dex.rs @@ -423,8 +423,7 @@ fn migration_1(_ctx: &MmArc) {} #[cfg(target_arch = "wasm32")] fn init_wasm_event_streaming(ctx: &MmArc) { - let event_streaming_config = ctx.event_streaming_configuration(); - if !event_streaming_config.disabled { + if let Some(event_streaming_config) = ctx.event_streaming_configuration() { ctx.spawner() .spawn(handle_worker_stream(ctx.clone(), event_streaming_config.worker_path)); } diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index 84eda03cc1..3fc1c5c333 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -12,10 +12,9 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result return handle_internal_error(err).await, }; - let event_streaming_config = ctx.event_streaming_configuration(); - if event_streaming_config.disabled { - return handle_internal_error("Event streaming is explicitly disabled".to_string()).await; - } + let Some(event_streaming_config) = ctx.event_streaming_configuration() else { + return handle_internal_error("Event streaming is disabled".to_string()).await; + }; let client_id = match request.uri().query().and_then(|query| { query From 547fc2bb4c03980379cfd219995e1d5ffa199bde Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 12 Oct 2024 21:09:40 +0200 Subject: [PATCH 108/128] style & code suggestions by onur 1 --- .../eth/fee_estimation/eth_fee_events.rs | 1 + mm2src/coins/eth/fee_estimation/rpc.rs | 15 ++++------ .../tendermint/tendermint_tx_history_v2.rs | 5 ++-- mm2src/coins/utxo/tx_history_events.rs | 2 ++ mm2src/coins/utxo/utxo_tx_history_v2.rs | 6 ++-- mm2src/coins/z_coin/tx_history_events.rs | 2 ++ mm2src/coins/z_coin/z_balance_streaming.rs | 2 ++ mm2src/mm2_event_stream/src/event.rs | 8 +++-- .../src/lp_ordermatch/order_events.rs | 4 ++- mm2src/mm2_main/src/lp_swap/swap_events.rs | 4 ++- .../src/rpc/streaming_activations/balance.rs | 30 ++++++++++--------- .../src/rpc/streaming_activations/disable.rs | 4 +-- .../src/rpc/streaming_activations/mod.rs | 4 +-- .../src/rpc/streaming_activations/swaps.rs | 9 ++++-- .../src/event_streaming/sse_handler.rs | 7 +++-- mm2src/rpc_task/src/task.rs | 2 +- 16 files changed, 59 insertions(+), 46 deletions(-) diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index 852988ef9e..8d91790374 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -43,6 +43,7 @@ pub struct EthFeeEventStreamer { } impl EthFeeEventStreamer { + #[inline(always)] pub fn new(config: EthFeeStreamingConfig, coin: EthCoin) -> Self { Self { config, coin } } } diff --git a/mm2src/coins/eth/fee_estimation/rpc.rs b/mm2src/coins/eth/fee_estimation/rpc.rs index 111d8c7979..6fb3b84498 100644 --- a/mm2src/coins/eth/fee_estimation/rpc.rs +++ b/mm2src/coins/eth/fee_estimation/rpc.rs @@ -3,7 +3,7 @@ use super::ser::FeePerGasEstimated; use crate::{lp_coinfind, MmCoinEnum}; use common::HttpStatusCode; use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::mm_error::MmResult; +use mm2_err_handle::mm_error::{MmError, MmResult}; use http::StatusCode; use std::convert::TryFrom; @@ -36,13 +36,8 @@ pub async fn get_eth_estimated_fee_per_gas( ctx: MmArc, req: GetFeeEstimationRequest, ) -> MmResult { - let coin = lp_coinfind(&ctx, &req.coin) - .await - .map_err(GetFeeEstimationRequestError::Internal)? - .ok_or(GetFeeEstimationRequestError::CoinNotFound)?; - - match coin { - MmCoinEnum::EthCoin(coin) => { + match lp_coinfind(&ctx, &req.coin).await { + Ok(Some(MmCoinEnum::EthCoin(coin))) => { let use_simple = matches!(req.estimator_type, EstimatorType::Simple); let fee = coin .get_eip1559_gas_fee(use_simple) @@ -52,6 +47,8 @@ pub async fn get_eth_estimated_fee_per_gas( FeePerGasEstimated::try_from(fee).map_err(|e| GetFeeEstimationRequestError::Internal(e.to_string()))?; Ok(ser_fee) }, - _ => Err(GetFeeEstimationRequestError::CoinNotSupported)?, + Ok(Some(_)) => MmError::err(GetFeeEstimationRequestError::CoinNotSupported), + Ok(None) => MmError::err(GetFeeEstimationRequestError::CoinNotFound), + Err(e) => MmError::err(GetFeeEstimationRequestError::Internal(e)), } } diff --git a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs index ece9b6a89c..9e83f707cc 100644 --- a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs +++ b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs @@ -588,6 +588,7 @@ where "could not get rpc client" ); + let streaming_manager = coin.get_ctx().unwrap().event_stream_manager.clone(); loop { let response = try_or_return_stopped_as_err!( client @@ -755,9 +756,7 @@ where log::debug!("Tx '{}' successfully parsed.", tx.hash); } - coin.get_ctx() - .unwrap() - .event_stream_manager + streaming_manager .send_fn(&TxHistoryEventStreamer::derive_streamer_id(coin.ticker()), || { tx_details.clone() }) diff --git a/mm2src/coins/utxo/tx_history_events.rs b/mm2src/coins/utxo/tx_history_events.rs index 902fb5baeb..c336e6fbb0 100644 --- a/mm2src/coins/utxo/tx_history_events.rs +++ b/mm2src/coins/utxo/tx_history_events.rs @@ -10,8 +10,10 @@ pub struct TxHistoryEventStreamer { } impl TxHistoryEventStreamer { + #[inline(always)] pub fn new(coin: String) -> Self { Self { coin } } + #[inline(always)] pub fn derive_streamer_id(coin: &str) -> String { format!("TX_HISTORY:{coin}") } } diff --git a/mm2src/coins/utxo/utxo_tx_history_v2.rs b/mm2src/coins/utxo/utxo_tx_history_v2.rs index 8670ceaac8..c60588e226 100644 --- a/mm2src/coins/utxo/utxo_tx_history_v2.rs +++ b/mm2src/coins/utxo/utxo_tx_history_v2.rs @@ -590,6 +590,7 @@ where let my_addresses = try_or_stop_unknown!(ctx.coin.my_addresses().await, "Error on getting my addresses"); + let streaming_manager = ctx.coin.get_ctx().unwrap().event_stream_manager.clone(); for (tx_hash, height) in self.all_tx_ids_with_height { let tx_hash_string = format!("{:02x}", tx_hash); match ctx.storage.history_has_tx_hash(&wallet_id, &tx_hash_string).await { @@ -621,10 +622,7 @@ where }, }; - ctx.coin - .get_ctx() - .unwrap() - .event_stream_manager + streaming_manager .send_fn(&TxHistoryEventStreamer::derive_streamer_id(ctx.coin.ticker()), || { tx_details.clone() }) diff --git a/mm2src/coins/z_coin/tx_history_events.rs b/mm2src/coins/z_coin/tx_history_events.rs index 9097d581c7..beb57a2b1a 100644 --- a/mm2src/coins/z_coin/tx_history_events.rs +++ b/mm2src/coins/z_coin/tx_history_events.rs @@ -19,8 +19,10 @@ pub struct ZCoinTxHistoryEventStreamer { } impl ZCoinTxHistoryEventStreamer { + #[inline(always)] pub fn new(coin: ZCoin) -> Self { Self { coin } } + #[inline(always)] pub fn derive_streamer_id(coin: &str) -> String { format!("TX_HISTORY:{coin}") } } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 2b29f093ed..18b2c6ef38 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -19,8 +19,10 @@ pub struct ZCoinBalanceEventStreamer { } impl ZCoinBalanceEventStreamer { + #[inline(always)] pub fn new(coin: ZCoin) -> Self { Self { coin } } + #[inline(always)] pub fn derive_streamer_id(coin: &str) -> String { format!("BALANCE:{coin}") } } diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index 93b1918d6a..57931b9817 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -2,9 +2,9 @@ use serde_json::Value as Json; // Note `Event` shouldn't be `Clone`able, but rather Arc/Rc wrapped and then shared. // This is only for testing. +/// Multi-purpose/generic event type that can easily be used over the event streaming #[cfg_attr(test, derive(Clone, Debug, PartialEq))] #[derive(Default)] -/// Multi-purpose/generic event type that can easily be used over the event streaming pub struct Event { /// The type of the event (balance, network, swap, etc...). event_type: String, @@ -16,7 +16,7 @@ pub struct Event { impl Event { /// Creates a new `Event` instance with the specified event type and message. - #[inline] + #[inline(always)] pub fn new(streamer_id: String, message: Json) -> Self { Self { event_type: streamer_id, @@ -26,7 +26,7 @@ impl Event { } /// Create a new error `Event` instance with the specified error event type and message. - #[inline] + #[inline(always)] pub fn err(streamer_id: String, message: Json) -> Self { Self { event_type: streamer_id, @@ -36,8 +36,10 @@ impl Event { } /// Returns the `event_type` (the ID of the streamer firing this event). + #[inline(always)] pub fn origin(&self) -> &str { &self.event_type } + /// Returns the event type and message as a pair. pub fn get(&self) -> (String, Json) { let prefix = if self.error { "ERROR:" } else { "" }; (format!("{prefix}{}", self.event_type), self.message.clone()) diff --git a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs index 0ba4dc3905..547ee7df4e 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/order_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/order_events.rs @@ -8,9 +8,11 @@ use futures::StreamExt; pub struct OrderStatusStreamer; impl OrderStatusStreamer { + #[inline(always)] pub fn new() -> Self { Self } - pub fn derive_streamer_id() -> &'static str { "ORDER_STATUS" } + #[inline(always)] + pub const fn derive_streamer_id() -> &'static str { "ORDER_STATUS" } } #[derive(Serialize)] diff --git a/mm2src/mm2_main/src/lp_swap/swap_events.rs b/mm2src/mm2_main/src/lp_swap/swap_events.rs index a7dcfef3b8..7f4aaa90eb 100644 --- a/mm2src/mm2_main/src/lp_swap/swap_events.rs +++ b/mm2src/mm2_main/src/lp_swap/swap_events.rs @@ -12,9 +12,11 @@ use uuid::Uuid; pub struct SwapStatusStreamer; impl SwapStatusStreamer { + #[inline(always)] pub fn new() -> Self { Self } - pub fn derive_streamer_id() -> &'static str { "SWAP_STATUS" } + #[inline(always)] + pub const fn derive_streamer_id() -> &'static str { "SWAP_STATUS" } } #[derive(Serialize)] diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs index fa49f0a8d6..76f9d594e1 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/balance.rs @@ -49,19 +49,32 @@ pub async fn enable_balance( .map_err(BalanceStreamingRequestError::Internal)? .ok_or(BalanceStreamingRequestError::CoinNotFound)?; + match coin { + MmCoinEnum::EthCoin(_) => (), + MmCoinEnum::ZCoin(_) + | MmCoinEnum::UtxoCoin(_) + | MmCoinEnum::Bch(_) + | MmCoinEnum::QtumCoin(_) + | MmCoinEnum::Tendermint(_) => { + if req.config.is_some() { + Err(BalanceStreamingRequestError::EnableError( + "Invalid config provided. No config needed".to_string(), + ))? + } + }, + _ => Err(BalanceStreamingRequestError::CoinNotSupported)?, + } + let enable_result = match coin { MmCoinEnum::UtxoCoin(coin) => { - check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Bch(coin) => { - check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::QtumCoin(coin) => { - check_empty_config(&req.config)?; let streamer = UtxoBalanceEventStreamer::new(coin.clone().into()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, @@ -71,12 +84,10 @@ pub async fn enable_balance( ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::ZCoin(coin) => { - check_empty_config(&req.config)?; let streamer = ZCoinBalanceEventStreamer::new(coin.clone()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, MmCoinEnum::Tendermint(coin) => { - check_empty_config(&req.config)?; let streamer = TendermintBalanceEventStreamer::new(coin.clone()); ctx.event_stream_manager.add(client_id, streamer, coin.spawner()).await }, @@ -87,12 +98,3 @@ pub async fn enable_balance( .map(EnableStreamingResponse::new) .map_to_mm(|e| BalanceStreamingRequestError::EnableError(format!("{e:?}"))) } - -fn check_empty_config(config: &Option) -> MmResult<(), BalanceStreamingRequestError> { - if config.is_some() { - Err(BalanceStreamingRequestError::EnableError( - "Invalid config provided. No config needed".to_string(), - ))? - } - Ok(()) -} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs b/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs index ab1155378e..9643e9e652 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/disable.rs @@ -8,15 +8,15 @@ use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; use http::StatusCode; -#[derive(Deserialize)] /// The request used for any event streaming deactivation. +#[derive(Deserialize)] pub struct DisableStreamingRequest { pub client_id: u64, pub streamer_id: String, } -#[derive(Serialize)] /// The success/ok response for any event streaming deactivation request. +#[derive(Serialize)] pub struct DisableStreamingResponse { result: &'static str, } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index c0a9bdf800..607876725f 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -17,9 +17,9 @@ pub use orders::*; pub use swaps::*; pub use tx_history::*; -#[derive(Deserialize)] /// The general request for enabling any streamer. /// `client_id` is common in each request, other data is request-specific. +#[derive(Deserialize)] pub struct EnableStreamingRequest { // If the client ID isn't included, assume it's 0. #[serde(default)] @@ -28,8 +28,8 @@ pub struct EnableStreamingRequest { inner: T, } -#[derive(Serialize)] /// The success/ok response for any event streaming activation request. +#[derive(Serialize)] pub struct EnableStreamingResponse { pub streamer_id: String, // TODO: If the the streamer was already running, it is probably running with different configuration. diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs index 3c3cb5d248..3d4aa2b93e 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/swaps.rs @@ -14,16 +14,19 @@ pub enum SwapStatusStreamingRequestError { } impl HttpStatusCode for SwapStatusStreamingRequestError { - fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } + fn status_code(&self) -> StatusCode { + match self { + SwapStatusStreamingRequestError::EnableError(_) => StatusCode::BAD_REQUEST, + } + } } pub async fn enable_swap_status( ctx: MmArc, req: EnableStreamingRequest<()>, ) -> MmResult { - let swap_status_streamer = SwapStatusStreamer::new(); ctx.event_stream_manager - .add(req.client_id, swap_status_streamer, ctx.spawner()) + .add(req.client_id, SwapStatusStreamer::new(), ctx.spawner()) .await .map(EnableStreamingResponse::new) .map_to_mm(|e| SwapStatusStreamingRequestError::EnableError(format!("{e:?}"))) diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index 3fc1c5c333..3dfa6264d7 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -1,3 +1,4 @@ +use http::header::{ACCESS_CONTROL_ALLOW_ORIGIN, CACHE_CONTROL, CONTENT_TYPE}; use hyper::{body::Bytes, Body, Request, Response}; use mm2_core::mm_ctx::MmArc; use serde_json::json; @@ -49,10 +50,10 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Result) -> Result>; } -#[derive(Deserialize)] /// The general request for initializing an RPC Task. /// /// `client_id` is used to identify the client to which the task should stream out update events /// to and is common in each request. Other data is request-specific. +#[derive(Deserialize)] pub struct RpcInitReq { // If the client ID isn't included, assume it's 0. #[serde(default)] From 3c5b1c6bcec84219ac2d6243649ef584936920c7 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 12 Oct 2024 21:15:25 +0200 Subject: [PATCH 109/128] style & code suggestions by onur 2 --- mm2src/mm2_event_stream/src/manager.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 38552836f4..e1c549424a 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -306,17 +306,16 @@ impl StreamingManager { /// In this case, we need to remove the streamer and de-list it from all clients. fn remove_streamer_if_down(&self, streamer_id: &str) { let mut this = self.write(); - if let Some(streamer_info) = this.streamers.get_mut(streamer_id) { - if streamer_info.is_down() { - // Remove the streamer from our registry. - if let Some(streamer_info) = this.streamers.remove(streamer_id) { - // And remove the streamer from all clients listening to it. - for client_id in streamer_info.clients { - if let Some(info) = this.clients.get_mut(&client_id) { - info.remove_streamer(streamer_id); - } - } - } + let Some(streamer_info) = this.streamers.get(streamer_id) else { return }; + if !streamer_info.is_down() { + return; + } + // Remove the streamer from our registry. + let Some(streamer_info) = this.streamers.remove(streamer_id) else { return }; + // And remove the streamer from all clients listening to it. + for client_id in streamer_info.clients { + if let Some(info) = this.clients.get_mut(&client_id) { + info.remove_streamer(streamer_id); } } } From 6e7c47d5ffcb0b4df7199b055eec4b4a660313ae Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 12 Oct 2024 21:28:03 +0200 Subject: [PATCH 110/128] don't expose the streamer spawn function so not in the trait methods --- mm2src/mm2_event_stream/src/manager.rs | 4 +- mm2src/mm2_event_stream/src/streamer.rs | 126 ++++++++++++------------ 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index e1c549424a..5c49a6ca1f 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -2,6 +2,7 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use crate::streamer::spawn; use crate::{Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; @@ -144,8 +145,7 @@ impl StreamingManager { } // Spawn a new streamer. - let (shutdown, data_in) = streamer - .spawn(spawner, self.clone()) + let (shutdown, data_in) = spawn(streamer, spawner, self.clone()) .await .map_err(StreamingManagerError::SpawnError)?; let streamer_info = StreamerInfo::new(data_in, shutdown); diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index d41ed920a9..9c42f4d6ec 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -37,69 +37,70 @@ where ready_tx: oneshot::Sender>, data_rx: impl StreamHandlerInput, ); +} - /// Spawns the `Self::handle` in a separate thread. - /// - /// Returns a `oneshot::Sender` to shutdown the handler and an optional `mpsc::UnboundedSender` - /// to send data to the handler. - /// - /// This method should not be overridden. - async fn spawn( - self, - spawner: WeakSpawner, - streaming_manager: StreamingManager, - ) -> Result<(oneshot::Sender<()>, Option>>), String> { - let streamer_id = self.streamer_id(); - info!("Spawning event streamer: {streamer_id}"); - - // A oneshot channel to receive the initialization status of the handler through. - let (tx_ready, ready_rx) = oneshot::channel(); - // A oneshot channel to shutdown the handler. - let (tx_shutdown, rx_shutdown) = oneshot::channel::<()>(); - // An unbounded channel to send data to the handler. - let (any_data_sender, any_data_receiver) = mpsc::unbounded::>(); - // A middleware to cast the data of type `Box` to the actual input datatype of this streamer. - let data_receiver = any_data_receiver.filter_map({ - let streamer_id = streamer_id.clone(); - move |any_input_data| { - let streamer_id = streamer_id.clone(); - Box::pin(async move { - if let Ok(input_data) = any_input_data.downcast() { - Some(*input_data) - } else { - error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); - None - } - }) - } - }); - - let handler_with_shutdown = { +/// Spawns the `EventStreamer::handle` in a separate thread. +/// +/// Returns a `oneshot::Sender` to shutdown the handler and an optional `mpsc::UnboundedSender` +/// to send data to the handler. +pub(crate) async fn spawn( + streamer: S, + spawner: WeakSpawner, + streaming_manager: StreamingManager, +) -> Result<(oneshot::Sender<()>, Option>>), String> +where + S: EventStreamer, +{ + let streamer_id = streamer.streamer_id(); + info!("Spawning event streamer: {streamer_id}"); + + // A oneshot channel to receive the initialization status of the handler through. + let (tx_ready, ready_rx) = oneshot::channel(); + // A oneshot channel to shutdown the handler. + let (tx_shutdown, rx_shutdown) = oneshot::channel::<()>(); + // An unbounded channel to send data to the handler. + let (any_data_sender, any_data_receiver) = mpsc::unbounded::>(); + // A middleware to cast the data of type `Box` to the actual input datatype of this streamer. + let data_receiver = any_data_receiver.filter_map({ + let streamer_id = streamer_id.clone(); + move |any_input_data| { let streamer_id = streamer_id.clone(); - async move { - select! { - _ = rx_shutdown.fuse() => { - info!("Manually shutting down event streamer: {streamer_id}.") - } - _ = self.handle(Broadcaster::new(streaming_manager), tx_ready, data_receiver).fuse() => {} + Box::pin(async move { + if let Ok(input_data) = any_input_data.downcast() { + Some(*input_data) + } else { + error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::<::DataInType>()); + None + } + }) + } + }); + + let handler_with_shutdown = { + let streamer_id = streamer_id.clone(); + async move { + select! { + _ = rx_shutdown.fuse() => { + info!("Manually shutting down event streamer: {streamer_id}.") } + _ = streamer.handle(Broadcaster::new(streaming_manager), tx_ready, data_receiver).fuse() => {} } - }; - let settings = AbortSettings::info_on_abort(format!("{streamer_id} streamer has stopped.")); - spawner.spawn_with_settings(handler_with_shutdown, settings); - - ready_rx.await.unwrap_or_else(|e| { - Err(format!( - "The handler was aborted before sending event initialization status: {e}" - )) - })?; - - // If the handler takes no input data, return `None` for the data sender. - if any::TypeId::of::() == any::TypeId::of::() { - Ok((tx_shutdown, None)) - } else { - Ok((tx_shutdown, Some(any_data_sender))) } + }; + let settings = AbortSettings::info_on_abort(format!("{streamer_id} streamer has stopped.")); + spawner.spawn_with_settings(handler_with_shutdown, settings); + + ready_rx.await.unwrap_or_else(|e| { + Err(format!( + "The handler was aborted before sending event initialization status: {e}" + )) + })?; + + // If the handler takes no input data, return `None` for the data sender. + if any::TypeId::of::<::DataInType>() == any::TypeId::of::() { + Ok((tx_shutdown, None)) + } else { + Ok((tx_shutdown, Some(any_data_sender))) } } @@ -198,8 +199,7 @@ mod tests { async fn test_spawn_periodic_streamer() { let system = AbortableQueue::default(); // Spawn the periodic streamer. - let (_, data_in) = PeriodicStreamer - .spawn(system.weak_spawner(), StreamingManager::default()) + let (_, data_in) = spawn(PeriodicStreamer, system.weak_spawner(), StreamingManager::default()) .await .unwrap(); // Periodic streamer shouldn't be ingesting any input. @@ -210,8 +210,7 @@ mod tests { async fn test_spawn_reactive_streamer() { let system = AbortableQueue::default(); // Spawn the reactive streamer. - let (_, data_in) = ReactiveStreamer - .spawn(system.weak_spawner(), StreamingManager::default()) + let (_, data_in) = spawn(ReactiveStreamer, system.weak_spawner(), StreamingManager::default()) .await .unwrap(); // Reactive streamer should be ingesting some input. @@ -222,8 +221,7 @@ mod tests { async fn test_spawn_erroring_streamer() { let system = AbortableQueue::default(); // Try to spawn the erroring streamer. - let err = InitErrorStreamer - .spawn(system.weak_spawner(), StreamingManager::default()) + let err = spawn(InitErrorStreamer, system.weak_spawner(), StreamingManager::default()) .await .unwrap_err(); // The streamer should return an error. From 8d1cffdc4987f86b5d22885b81150159987a1917 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sun, 13 Oct 2024 10:24:51 +0200 Subject: [PATCH 111/128] use future::ready right away in streamer::spawn since we don't use any async computations here anyway --- mm2src/mm2_event_stream/src/streamer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 9c42f4d6ec..beb9a66635 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -6,7 +6,7 @@ use common::log::{error, info}; use async_trait::async_trait; use futures::channel::{mpsc, oneshot}; -use futures::{select, FutureExt, Stream, StreamExt}; +use futures::{future, select, FutureExt, Stream, StreamExt}; /// A marker to indicate that the event streamer doesn't take any input data. pub struct NoDataIn; @@ -65,14 +65,14 @@ where let streamer_id = streamer_id.clone(); move |any_input_data| { let streamer_id = streamer_id.clone(); - Box::pin(async move { + future::ready( if let Ok(input_data) = any_input_data.downcast() { Some(*input_data) } else { error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::<::DataInType>()); None } - }) + ) } }); From c1101c11c5082a73b21f4f1c6045c00c4f09eef1 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 15 Oct 2024 12:13:35 +0200 Subject: [PATCH 112/128] stream orderbook reactively this streams out the orderbook to subbed clients as the orders arrive to mm2. the model used for this case is a different streamer for each coin pair requested for streaming, so (btc, ltc) has a streamer, (btc, kmd) has another, and (ltc, btc) also has another. we might want to combine the first and last cases (btc, ltc) & (ltc, btc) into a single streamer? but then we would want the streamer to have a deterministic name/id (based on lexagraphical order), or tolerate different names for the same streamer (might be confusing and also harder to impl). --- mm2src/mm2_main/src/lp_ordermatch.rs | 30 +++++++++-- .../src/lp_ordermatch/orderbook_events.rs | 51 +++++++++++++++++++ .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 1 + .../src/rpc/streaming_activations/mod.rs | 2 + .../rpc/streaming_activations/orderbook.rs | 37 ++++++++++++++ 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs create mode 100644 mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index 1034966702..eeac303406 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -42,6 +42,7 @@ use http::Response; use keys::{AddressFormat, KeyPair}; use mm2_core::mm_ctx::{from_ctx, MmArc, MmWeak}; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_libp2p::{decode_signed, encode_and_sign, encode_message, pub_sub_topic, PublicKey, TopicHash, TopicPrefix, TOPIC_SEPARATOR}; use mm2_metrics::mm_gauge; @@ -55,6 +56,7 @@ use my_orders_storage::{delete_my_maker_order, delete_my_taker_order, save_maker MyOrdersHistory, MyOrdersStorage}; use num_traits::identities::Zero; use order_events::{OrderStatusEvent, OrderStatusStreamer}; +use orderbook_events::{OrderbookItemChangeEvent, OrderbookStreamer}; use parking_lot::Mutex as PaMutex; use rpc::v1::types::H256 as H256Json; use serde_json::{self as json, Value as Json}; @@ -106,6 +108,7 @@ mod new_protocol; pub(crate) mod order_events; mod order_requests_tracker; mod orderbook_depth; +pub(crate) mod orderbook_events; mod orderbook_rpc; #[cfg(all(test, not(target_arch = "wasm32")))] #[path = "ordermatch_tests.rs"] @@ -2512,6 +2515,8 @@ struct Orderbook { /// MemoryDB instance to store Patricia Tries data memory_db: MemoryDB, my_p2p_pubkeys: HashSet, + /// A copy of the streaming manager to stream orderbook events out. + streaming_manager: StreamingManager, } impl Default for Orderbook { @@ -2527,6 +2532,7 @@ impl Default for Orderbook { topics_subscribed_to: HashMap::default(), memory_db: MemoryDB::default(), my_p2p_pubkeys: HashSet::default(), + streaming_manager: Default::default(), } } } @@ -2534,6 +2540,13 @@ impl Default for Orderbook { fn hashed_null_node() -> TrieHash { ::hashed_null_node() } impl Orderbook { + fn new(streaming_manager: StreamingManager) -> Orderbook { + Orderbook { + streaming_manager, + ..Default::default() + } + } + fn find_order_by_uuid_and_pubkey(&self, uuid: &Uuid, from_pubkey: &str) -> Option { self.order_set.get(uuid).and_then(|order| { if order.pubkey == from_pubkey { @@ -2637,6 +2650,11 @@ impl Orderbook { .or_insert_with(HashSet::new) .insert(order.uuid); + self.streaming_manager + .send_fn(&OrderbookStreamer::derive_streamer_id(&order.base, &order.rel), || { + OrderbookItemChangeEvent::NewOrUpdatedItem(order.clone().into()) + }) + .ok(); self.order_set.insert(order.uuid, order); } @@ -2694,6 +2712,12 @@ impl Orderbook { next_root: *pair_state, }); } + + self.streaming_manager + .send_fn(&OrderbookStreamer::derive_streamer_id(&order.base, &order.rel), || { + OrderbookItemChangeEvent::RemovedItem(order.uuid) + }) + .ok(); Some(order) } @@ -2795,7 +2819,7 @@ pub fn init_ordermatch_context(ctx: &MmArc) -> OrdermatchInitResult<()> { let ordermatch_context = OrdermatchContext { maker_orders_ctx: PaMutex::new(MakerOrdersContext::new(ctx)?), my_taker_orders: Default::default(), - orderbook: Default::default(), + orderbook: PaMutex::new(Orderbook::new(ctx.event_stream_manager.clone())), pending_maker_reserved: Default::default(), orderbook_tickers, original_tickers, @@ -2825,7 +2849,7 @@ impl OrdermatchContext { Ok(OrdermatchContext { maker_orders_ctx: PaMutex::new(try_s!(MakerOrdersContext::new(ctx))), my_taker_orders: Default::default(), - orderbook: Default::default(), + orderbook: PaMutex::new(Orderbook::new(ctx.event_stream_manager.clone())), pending_maker_reserved: Default::default(), orderbook_tickers: Default::default(), original_tickers: Default::default(), @@ -4039,7 +4063,7 @@ pub async fn lp_auto_buy( /// Orderbook Item P2P message /// DO NOT CHANGE - it will break backwards compatibility #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -struct OrderbookP2PItem { +pub struct OrderbookP2PItem { pubkey: String, base: String, rel: String, diff --git a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs new file mode 100644 index 0000000000..5773c6fc7e --- /dev/null +++ b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs @@ -0,0 +1,51 @@ +use super::OrderbookP2PItem; +use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; + +use async_trait::async_trait; +use futures::channel::oneshot; +use futures::StreamExt; +use uuid::Uuid; + +pub struct OrderbookStreamer { + base: String, + rel: String, +} + +impl OrderbookStreamer { + pub fn new(base: String, rel: String) -> Self { Self { base, rel } } + + pub fn derive_streamer_id(base: &str, rel: &str) -> String { format!("ORDERBOOK_UPDATE/{base}:{rel}") } +} + +#[derive(Serialize)] +#[serde(tag = "order_type", content = "order_data")] +pub enum OrderbookItemChangeEvent { + /// New or updated orderbook item. + NewOrUpdatedItem(OrderbookP2PItem), + /// Removed orderbook item (only UUID is relevant in this case). + RemovedItem(Uuid), +} + +#[async_trait] +impl EventStreamer for OrderbookStreamer { + type DataInType = OrderbookItemChangeEvent; + + fn streamer_id(&self) -> String { Self::derive_streamer_id(&self.base, &self.rel) } + + async fn handle( + self, + broadcaster: Broadcaster, + ready_tx: oneshot::Sender>, + mut data_rx: impl StreamHandlerInput, + ) { + ready_tx + .send(Ok(())) + .expect("Receiver is dropped, which should never happen."); + + while let Some(orderbook_update) = data_rx.next().await { + let event_data = serde_json::to_value(orderbook_update).expect("Serialization shouldn't fail."); + let event = Event::new(self.streamer_id(), event_data); + broadcaster.broadcast(event); + } + } +} diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 862e8f6023..3fbb0a5215 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -345,6 +345,7 @@ async fn rpc_streaming_dispatcher( "swap_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_swap_status).await, "order_status::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_order_status).await, "tx_history::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_tx_history).await, + "orderbook::enable" => handle_mmrpc(ctx, request, streaming_activations::enable_orderbook).await, "disable" => handle_mmrpc(ctx, request, streaming_activations::disable_streamer).await, _ => MmError::err(DispatcherError::NoSuchMethod), } diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs index 607876725f..05d2848f97 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/mod.rs @@ -3,6 +3,7 @@ mod disable; mod fee_estimation; mod heartbeat; mod network; +mod orderbook; mod orders; mod swaps; mod tx_history; @@ -13,6 +14,7 @@ pub use disable::*; pub use fee_estimation::*; pub use heartbeat::*; pub use network::*; +pub use orderbook::*; pub use orders::*; pub use swaps::*; pub use tx_history::*; diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs new file mode 100644 index 0000000000..37c3e5d35e --- /dev/null +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs @@ -0,0 +1,37 @@ +//! RPC activation and deactivation of the orderbook streamer. +use super::EnableStreamingResponse; +use crate::lp_ordermatch::orderbook_events::OrderbookStreamer; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::{map_to_mm::MapToMmResult, mm_error::MmResult}; + +use common::HttpStatusCode; +use http::StatusCode; + +#[derive(Deserialize)] +pub struct EnableOrderbookStreamingRequest { + pub client_id: u64, + pub base: String, + pub rel: String, +} + +#[derive(Display, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum OrderbookStreamingRequestError { + EnableError(String), +} + +impl HttpStatusCode for OrderbookStreamingRequestError { + fn status_code(&self) -> StatusCode { StatusCode::BAD_REQUEST } +} + +pub async fn enable_orderbook( + ctx: MmArc, + req: EnableOrderbookStreamingRequest, +) -> MmResult { + let order_status_streamer = OrderbookStreamer::new(req.base, req.rel); + ctx.event_stream_manager + .add(req.client_id, order_status_streamer, ctx.spawner()) + .await + .map(EnableStreamingResponse::new) + .map_to_mm(|e| OrderbookStreamingRequestError::EnableError(format!("{e:?}"))) +} From a1918eca4b1524caa6562c57df3b71f00c04a2c1 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Tue, 15 Oct 2024 12:26:02 +0200 Subject: [PATCH 113/128] box orderbook update for enum balancing --- mm2src/mm2_main/src/lp_ordermatch.rs | 2 +- mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_main/src/lp_ordermatch.rs b/mm2src/mm2_main/src/lp_ordermatch.rs index eeac303406..256f103bf6 100644 --- a/mm2src/mm2_main/src/lp_ordermatch.rs +++ b/mm2src/mm2_main/src/lp_ordermatch.rs @@ -2652,7 +2652,7 @@ impl Orderbook { self.streaming_manager .send_fn(&OrderbookStreamer::derive_streamer_id(&order.base, &order.rel), || { - OrderbookItemChangeEvent::NewOrUpdatedItem(order.clone().into()) + OrderbookItemChangeEvent::NewOrUpdatedItem(Box::new(order.clone().into())) }) .ok(); self.order_set.insert(order.uuid, order); diff --git a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs index 5773c6fc7e..f5eaeae342 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs @@ -20,8 +20,9 @@ impl OrderbookStreamer { #[derive(Serialize)] #[serde(tag = "order_type", content = "order_data")] pub enum OrderbookItemChangeEvent { + // NOTE(clippy): This is box-ed due to in-balance of the size of enum variants. /// New or updated orderbook item. - NewOrUpdatedItem(OrderbookP2PItem), + NewOrUpdatedItem(Box), /// Removed orderbook item (only UUID is relevant in this case). RemovedItem(Uuid), } From 590b2a185721356e3284e0099f5214d548ebbbc9 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 7 Sep 2024 14:16:28 +0300 Subject: [PATCH 114/128] auto subscribe for orderbook topic when orderbook streaming is enabled this creates a side-effect which might not be favoured. --- .../src/lp_ordermatch/orderbook_events.rs | 48 +++++++++++++++++-- .../rpc/streaming_activations/orderbook.rs | 2 +- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs index f5eaeae342..779823d240 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs @@ -1,4 +1,6 @@ -use super::OrderbookP2PItem; +use super::{subscribe_to_orderbook_topic, OrderbookP2PItem}; +use coins::{is_wallet_only_ticker, lp_coinfind}; +use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; use async_trait::async_trait; @@ -7,12 +9,13 @@ use futures::StreamExt; use uuid::Uuid; pub struct OrderbookStreamer { + ctx: MmArc, base: String, rel: String, } impl OrderbookStreamer { - pub fn new(base: String, rel: String) -> Self { Self { base, rel } } + pub fn new(ctx: MmArc, base: String, rel: String) -> Self { Self { ctx, base, rel } } pub fn derive_streamer_id(base: &str, rel: &str) -> String { format!("ORDERBOOK_UPDATE/{base}:{rel}") } } @@ -39,9 +42,18 @@ impl EventStreamer for OrderbookStreamer { ready_tx: oneshot::Sender>, mut data_rx: impl StreamHandlerInput, ) { - ready_tx - .send(Ok(())) - .expect("Receiver is dropped, which should never happen."); + const RECEIVER_DROPPED_MSG: &str = "Receiver is dropped, which should never happen."; + if let Err(err) = sanity_checks(&self.ctx, &self.base, &self.rel).await { + ready_tx.send(Err(err.clone())).expect(RECEIVER_DROPPED_MSG); + panic!("{}", err); + } + // We need to subscribe to the orderbook, otherwise we won't get any updates from the P2P network. + if let Err(err) = subscribe_to_orderbook_topic(&self.ctx, &self.base, &self.rel, false).await { + let err = format!("Subscribing to orderbook topic failed: {err:?}"); + ready_tx.send(Err(err.clone())).expect(RECEIVER_DROPPED_MSG); + panic!("{}", err); + } + ready_tx.send(Ok(())).expect(RECEIVER_DROPPED_MSG); while let Some(orderbook_update) = data_rx.next().await { let event_data = serde_json::to_value(orderbook_update).expect("Serialization shouldn't fail."); @@ -50,3 +62,29 @@ impl EventStreamer for OrderbookStreamer { } } } + +async fn sanity_checks(ctx: &MmArc, base: &str, rel: &str) -> Result<(), String> { + lp_coinfind(ctx, base) + .await + .map_err(|e| format!("Coin {base} not found: {e}"))?; + if is_wallet_only_ticker(ctx, base) { + return Err(format!("Coin {base} is wallet-only.")); + } + lp_coinfind(ctx, rel) + .await + .map_err(|e| format!("Coin {base} not found: {e}"))?; + if is_wallet_only_ticker(ctx, rel) { + return Err(format!("Coin {rel} is wallet-only.")); + } + Ok(()) +} + +impl Drop for OrderbookStreamer { + fn drop(&mut self) { + // FIXME(discuss): Do we want to unsubscribe from the orderbook topic when streaming is dropped? + + // Note that the client enables orderbook streaming for all enabled coins when they query for best + // orders. These could potentially be a lot of pairs! + // Also, in the dev branch, we seem to never unsubscribe from an orderbook topic after doing an orderbook RPC! + } +} diff --git a/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs b/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs index 37c3e5d35e..60805c4a54 100644 --- a/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs +++ b/mm2src/mm2_main/src/rpc/streaming_activations/orderbook.rs @@ -28,7 +28,7 @@ pub async fn enable_orderbook( ctx: MmArc, req: EnableOrderbookStreamingRequest, ) -> MmResult { - let order_status_streamer = OrderbookStreamer::new(req.base, req.rel); + let order_status_streamer = OrderbookStreamer::new(ctx.clone(), req.base, req.rel); ctx.event_stream_manager .add(req.client_id, order_status_streamer, ctx.spawner()) .await From 7fe3e5365e9f8543ad5baa88f953c846579bf05e Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 7 Sep 2024 14:27:44 +0300 Subject: [PATCH 115/128] merge base/rel & rel/base orderbook streamers into one streamer for their topic --- mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs index 779823d240..56e5ca2372 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs @@ -1,4 +1,4 @@ -use super::{subscribe_to_orderbook_topic, OrderbookP2PItem}; +use super::{orderbook_topic_from_base_rel, subscribe_to_orderbook_topic, OrderbookP2PItem}; use coins::{is_wallet_only_ticker, lp_coinfind}; use mm2_core::mm_ctx::MmArc; use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; @@ -17,7 +17,9 @@ pub struct OrderbookStreamer { impl OrderbookStreamer { pub fn new(ctx: MmArc, base: String, rel: String) -> Self { Self { ctx, base, rel } } - pub fn derive_streamer_id(base: &str, rel: &str) -> String { format!("ORDERBOOK_UPDATE/{base}:{rel}") } + pub fn derive_streamer_id(base: &str, rel: &str) -> String { + format!("ORDERBOOK_UPDATE/{}", orderbook_topic_from_base_rel(base, rel)) + } } #[derive(Serialize)] From d84bf39ce597b4498f48e35eaa6defc563e36432 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 19 Oct 2024 11:48:20 +0200 Subject: [PATCH 116/128] add a todo regarding orderbook streaming in no-login mode --- mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs index 56e5ca2372..b1954ac57f 100644 --- a/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs +++ b/mm2src/mm2_main/src/lp_ordermatch/orderbook_events.rs @@ -66,6 +66,7 @@ impl EventStreamer for OrderbookStreamer { } async fn sanity_checks(ctx: &MmArc, base: &str, rel: &str) -> Result<(), String> { + // TODO: This won't work with no-login mode. lp_coinfind(ctx, base) .await .map_err(|e| format!("Coin {base} not found: {e}"))?; From 52af46fe40523e897eaa0349c8398018cfb3435d Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 19 Oct 2024 12:19:18 +0200 Subject: [PATCH 117/128] review(dimxy): fix streamer spawner doc comment --- mm2src/mm2_event_stream/src/streamer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index beb9a66635..6055d05149 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -39,9 +39,9 @@ where ); } -/// Spawns the `EventStreamer::handle` in a separate thread. +/// Spawns the [`EventStreamer::handle`] in a separate task using [`WeakSpawner`]. /// -/// Returns a `oneshot::Sender` to shutdown the handler and an optional `mpsc::UnboundedSender` +/// Returns a [`oneshot::Sender`] to shutdown the handler and an optional [`mpsc::UnboundedSender`] /// to send data to the handler. pub(crate) async fn spawn( streamer: S, From 25a27b959ea9d3420860fa27fbe6c54623539e77 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 30 Oct 2024 20:45:36 +0100 Subject: [PATCH 118/128] post(1966): enable utxo balance events the streaming manager is passed up to ElectrumClientImpl, we could thoeretically pass it down the tree to each ElectrumConnection, but since these connections already have a ref to ElectrumClientImpl in their establish_connecion_loop lifetime, we don't really need to do so. --- mm2src/coins/utxo.rs | 4 +- .../utxo/rpc_clients/electrum_rpc/client.rs | 55 +++++++++++-------- .../rpc_clients/electrum_rpc/connection.rs | 32 ++++++----- .../electrum_rpc/event_handlers.rs | 45 --------------- mm2src/coins/utxo/utxo_balance_events.rs | 2 + .../utxo/utxo_builder/utxo_coin_builder.rs | 21 +++---- mm2src/coins/utxo/utxo_tests.rs | 13 +++-- mm2src/coins/utxo/utxo_wasm_tests.rs | 2 +- mm2src/coins/z_coin/z_balance_streaming.rs | 6 -- mm2src/coins/z_coin/z_coin_native_tests.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 6 +- 11 files changed, 71 insertions(+), 117 deletions(-) diff --git a/mm2src/coins/utxo.rs b/mm2src/coins/utxo.rs index 7dcf3eae5d..49c6070707 100644 --- a/mm2src/coins/utxo.rs +++ b/mm2src/coins/utxo.rs @@ -57,7 +57,7 @@ use common::{now_sec, now_sec_u32}; use crypto::{DerivationPath, HDPathToCoin, Secp256k1ExtendedPublicKey}; use derive_more::Display; #[cfg(not(target_arch = "wasm32"))] use dirs::home_dir; -use futures::channel::mpsc::{Receiver as AsyncReceiver, Sender as AsyncSender, UnboundedSender}; +use futures::channel::mpsc::{Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::compat::Future01CompatExt; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures01::Future; @@ -145,8 +145,6 @@ pub enum ScripthashNotification { SubscribeToAddresses(HashSet
), } -pub type ScripthashNotificationSender = Option>; - #[cfg(windows)] #[cfg(not(target_arch = "wasm32"))] fn get_special_folder_path() -> PathBuf { diff --git a/mm2src/coins/utxo/rpc_clients/electrum_rpc/client.rs b/mm2src/coins/utxo/rpc_clients/electrum_rpc/client.rs index ce0498cc31..84e39997e4 100644 --- a/mm2src/coins/utxo/rpc_clients/electrum_rpc/client.rs +++ b/mm2src/coins/utxo/rpc_clients/electrum_rpc/client.rs @@ -5,7 +5,7 @@ use super::connection_manager::ConnectionManager; use super::constants::{BLOCKCHAIN_HEADERS_SUB_ID, BLOCKCHAIN_SCRIPTHASH_SUB_ID, ELECTRUM_REQUEST_TIMEOUT, NO_FORCE_CONNECT_METHODS, SEND_TO_ALL_METHODS}; use super::electrum_script_hash; -use super::event_handlers::{ElectrumConnectionManagerNotifier, ElectrumScriptHashNotificationBridge}; +use super::event_handlers::ElectrumConnectionManagerNotifier; use super::rpc_responses::*; use crate::utxo::rpc_clients::ConcurrentRequestMap; @@ -43,14 +43,15 @@ use std::ops::Deref; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::Arc; +use crate::utxo::utxo_balance_events::UtxoBalanceEventStreamer; use async_trait::async_trait; -use futures::channel::mpsc::UnboundedSender; use futures::compat::Future01CompatExt; use futures::future::{join_all, FutureExt, TryFutureExt}; use futures::stream::FuturesUnordered; use futures::StreamExt; use futures01::Future; use itertools::Itertools; +use mm2_event_stream::StreamingManager; use serde_json::{self as json, Value as Json}; type ElectrumTxHistory = Vec; @@ -85,7 +86,8 @@ pub struct ElectrumClientImpl { /// in an `Arc` since they are shared outside `ElectrumClientImpl`. They are handed to each active /// `ElectrumConnection` to notify them about the events. event_handlers: Arc>>, - pub scripthash_notification_sender: Option>, + /// A streaming manager instance used to notify for Utxo balance events streamer. + streaming_manager: StreamingManager, abortable_system: AbortableQueue, } @@ -98,18 +100,10 @@ impl ElectrumClientImpl { fn try_new( client_settings: ElectrumClientSettings, block_headers_storage: BlockHeaderStorage, + streaming_manager: StreamingManager, abortable_system: AbortableQueue, mut event_handlers: Vec>, - scripthash_notification_sender: Option>, ) -> Result { - // This is used for balance event streaming implementation for UTXOs. - // Will be used for sending scripthash messages to trigger re-connections, re-fetching the balances, etc. - if let Some(scripthash_notification_sender) = scripthash_notification_sender.clone() { - event_handlers.push(Box::new(ElectrumScriptHashNotificationBridge { - scripthash_notification_sender, - })); - } - let connection_manager = ConnectionManager::try_new( client_settings.servers, client_settings.spawn_ping, @@ -132,7 +126,7 @@ impl ElectrumClientImpl { list_unspent_concurrent_map: ConcurrentRequestMap::new(), block_headers_storage, abortable_system, - scripthash_notification_sender, + streaming_manager, event_handlers: Arc::new(event_handlers), }) } @@ -142,16 +136,16 @@ impl ElectrumClientImpl { pub fn try_new_arc( client_settings: ElectrumClientSettings, block_headers_storage: BlockHeaderStorage, + streaming_manager: StreamingManager, abortable_system: AbortableQueue, event_handlers: Vec>, - scripthash_notification_sender: Option>, ) -> Result, String> { let client_impl = Arc::new(ElectrumClientImpl::try_new( client_settings, block_headers_storage, + streaming_manager, abortable_system, event_handlers, - scripthash_notification_sender, )?); // Initialize the connection manager. client_impl @@ -185,12 +179,25 @@ impl ElectrumClientImpl { /// Sends a list of addresses through the scripthash notification sender to subscribe to their scripthash notifications. pub fn subscribe_addresses(&self, addresses: HashSet
) -> Result<(), String> { - if let Some(sender) = &self.scripthash_notification_sender { - sender - .unbounded_send(ScripthashNotification::SubscribeToAddresses(addresses)) - .map_err(|e| ERRL!("Failed sending scripthash message. {}", e))?; - } + self.streaming_manager + .send( + &UtxoBalanceEventStreamer::derive_streamer_id(&self.coin_ticker), + ScripthashNotification::SubscribeToAddresses(addresses), + ) + .map_err(|e| ERRL!("Failed sending scripthash message. {:?}", e))?; + Ok(()) + } + /// Notifies the Utxo balance streamer of a new script hash balance change. + /// + /// The streamer will figure out which address this scripthash belongs to and will broadcast an notification to clients. + pub fn notify_triggered_hash(&self, script_hash: String) -> Result<(), String> { + self.streaming_manager + .send( + &UtxoBalanceEventStreamer::derive_streamer_id(&self.coin_ticker), + ScripthashNotification::Triggered(script_hash), + ) + .map_err(|e| ERRL!("Failed sending scripthash message. {:?}", e))?; Ok(()) } @@ -203,9 +210,9 @@ impl ElectrumClientImpl { pub fn with_protocol_version( client_settings: ElectrumClientSettings, block_headers_storage: BlockHeaderStorage, + streaming_manager: StreamingManager, abortable_system: AbortableQueue, event_handlers: Vec>, - scripthash_notification_sender: Option>, protocol_version: OrdRange, ) -> Result, String> { let client_impl = Arc::new(ElectrumClientImpl { @@ -213,9 +220,9 @@ impl ElectrumClientImpl { ..ElectrumClientImpl::try_new( client_settings, block_headers_storage, + streaming_manager, abortable_system, event_handlers, - scripthash_notification_sender, )? }); // Initialize the connection manager. @@ -272,15 +279,15 @@ impl ElectrumClient { client_settings: ElectrumClientSettings, event_handlers: Vec>, block_headers_storage: BlockHeaderStorage, + streaming_manager: StreamingManager, abortable_system: AbortableQueue, - scripthash_notification_sender: Option>, ) -> Result { let client = ElectrumClient(ElectrumClientImpl::try_new_arc( client_settings, block_headers_storage, + streaming_manager, abortable_system, event_handlers, - scripthash_notification_sender, )?); Ok(client) diff --git a/mm2src/coins/utxo/rpc_clients/electrum_rpc/connection.rs b/mm2src/coins/utxo/rpc_clients/electrum_rpc/connection.rs index 2b9a3ada48..691a76dd85 100644 --- a/mm2src/coins/utxo/rpc_clients/electrum_rpc/connection.rs +++ b/mm2src/coins/utxo/rpc_clients/electrum_rpc/connection.rs @@ -275,9 +275,9 @@ impl ElectrumConnection { } /// Process an incoming JSONRPC response from the electrum server. - fn process_electrum_response(&self, bytes: &[u8], event_handlers: &Vec>) { + fn process_electrum_response(&self, bytes: &[u8], client: &ElectrumClient) { // Inform the event handlers. - event_handlers.on_incoming_response(bytes); + client.event_handlers().on_incoming_response(bytes); // detect if we got standard JSONRPC response or subscription response as JSONRPC request #[derive(Deserialize)] @@ -308,8 +308,14 @@ impl ElectrumConnection { ElectrumRpcResponseEnum::BatchResponses(batch) => JsonRpcResponseEnum::Batch(batch), ElectrumRpcResponseEnum::SubscriptionNotification(req) => { match req.method.as_str() { - // NOTE: Sending a script hash notification is handled in it's own event handler. - BLOCKCHAIN_SCRIPTHASH_SUB_ID | BLOCKCHAIN_HEADERS_SUB_ID => {}, + BLOCKCHAIN_SCRIPTHASH_SUB_ID => { + if let Some(scripthash) = req.params.first().and_then(|s| s.as_str()) { + client.notify_triggered_hash(scripthash.to_string()).ok(); + } else { + error!("Notification must contain the script hash value, got: {req:?}"); + } + }, + BLOCKCHAIN_HEADERS_SUB_ID => {}, _ => { error!("Unexpected notification method: {}", req.method); }, @@ -329,18 +335,14 @@ impl ElectrumConnection { /// Process a bulk response from the electrum server. /// /// A bulk response is a response that contains multiple JSONRPC responses. - fn process_electrum_bulk_response( - &self, - bulk_response: &[u8], - event_handlers: &Vec>, - ) { + fn process_electrum_bulk_response(&self, bulk_response: &[u8], client: &ElectrumClient) { // We should split the received response because we can get several responses in bulk. let responses = bulk_response.split(|item| *item == b'\n'); for response in responses { // `split` returns empty slice if it ends with separator which is our case. if !response.is_empty() { - self.process_electrum_response(response, event_handlers) + self.process_electrum_response(response, client) } } } @@ -536,7 +538,7 @@ impl ElectrumConnection { #[cfg(not(target_arch = "wasm32"))] async fn recv_loop( connection: Arc, - event_handlers: Arc>>, + client: ElectrumClient, read: ReadHalf, last_response: Arc, ) -> ElectrumConnectionErr { @@ -555,7 +557,7 @@ impl ElectrumConnection { }; last_response.store(now_ms(), AtomicOrdering::Relaxed); - connection.process_electrum_bulk_response(buffer.as_bytes(), &event_handlers); + connection.process_electrum_bulk_response(buffer.as_bytes(), &client); buffer.clear(); } } @@ -563,7 +565,7 @@ impl ElectrumConnection { #[cfg(target_arch = "wasm32")] async fn recv_loop( connection: Arc, - event_handlers: Arc>>, + client: ElectrumClient, mut read: WsIncomingReceiver, last_response: Arc, ) -> ElectrumConnectionErr { @@ -572,7 +574,7 @@ impl ElectrumConnection { match response { Ok(bytes) => { last_response.store(now_ms(), AtomicOrdering::Relaxed); - connection.process_electrum_response(&bytes, &event_handlers); + connection.process_electrum_response(&bytes, &client); }, Err(e) => { error!("{address} error: {e:?}"); @@ -674,7 +676,7 @@ impl ElectrumConnection { let (read, write) = tokio::io::split(stream); #[cfg(target_arch = "wasm32")] let (read, write) = stream; - let recv_branch = Self::recv_loop(connection.clone(), event_handlers.clone(), read, last_response).boxed(); + let recv_branch = Self::recv_loop(connection.clone(), client.clone(), read, last_response).boxed(); // Branch 3: Send outgoing requests to the server. let (tx, rx) = mpsc::channel(0); diff --git a/mm2src/coins/utxo/rpc_clients/electrum_rpc/event_handlers.rs b/mm2src/coins/utxo/rpc_clients/electrum_rpc/event_handlers.rs index 27bd74b4d9..9db2ab93ec 100644 --- a/mm2src/coins/utxo/rpc_clients/electrum_rpc/event_handlers.rs +++ b/mm2src/coins/utxo/rpc_clients/electrum_rpc/event_handlers.rs @@ -1,51 +1,6 @@ use super::connection_manager::ConnectionManager; -use super::constants::BLOCKCHAIN_SCRIPTHASH_SUB_ID; -use crate::utxo::ScripthashNotification; use crate::RpcTransportEventHandler; -use common::jsonrpc_client::JsonRpcRequest; -use common::log::{error, warn}; - -use futures::channel::mpsc::UnboundedSender; -use serde_json::{self as json, Value as Json}; - -/// An `RpcTransportEventHandler` that forwards `ScripthashNotification`s to trigger balance updates. -/// -/// This handler hooks in `on_incoming_response` and looks for an electrum script hash notification to forward it. -pub struct ElectrumScriptHashNotificationBridge { - pub scripthash_notification_sender: UnboundedSender, -} - -impl RpcTransportEventHandler for ElectrumScriptHashNotificationBridge { - fn debug_info(&self) -> String { "ElectrumScriptHashNotificationBridge".into() } - - fn on_incoming_response(&self, data: &[u8]) { - if let Ok(raw_json) = json::from_slice::(data) { - // Try to parse the notification. A notification is sent as a JSON-RPC request. - if let Ok(notification) = json::from_value::(raw_json) { - // Only care about `BLOCKCHAIN_SCRIPTHASH_SUB_ID` notifications. - if notification.method.as_str() == BLOCKCHAIN_SCRIPTHASH_SUB_ID { - if let Some(scripthash) = notification.params.first().and_then(|s| s.as_str()) { - if let Err(e) = self - .scripthash_notification_sender - .unbounded_send(ScripthashNotification::Triggered(scripthash.to_string())) - { - error!("Failed sending script hash message. {e:?}"); - } - } else { - warn!("Notification must contain the script hash value, got: {notification:?}"); - } - }; - } - } - } - - fn on_connected(&self, _address: &str) -> Result<(), String> { Ok(()) } - - fn on_disconnected(&self, _address: &str) -> Result<(), String> { Ok(()) } - - fn on_outgoing_request(&self, _data: &[u8]) {} -} /// An `RpcTransportEventHandler` that notifies the `ConnectionManager` upon connections and disconnections. /// diff --git a/mm2src/coins/utxo/utxo_balance_events.rs b/mm2src/coins/utxo/utxo_balance_events.rs index 1643291045..5de90555e4 100644 --- a/mm2src/coins/utxo/utxo_balance_events.rs +++ b/mm2src/coins/utxo/utxo_balance_events.rs @@ -39,6 +39,8 @@ impl UtxoBalanceEventStreamer { coin: UtxoStandardCoin::from(utxo_arc), } } + + pub fn derive_streamer_id(coin: &str) -> String { format!("BALANCE:{coin}") } } #[async_trait] diff --git a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs index 74b00273ed..b916afc232 100644 --- a/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs +++ b/mm2src/coins/utxo/utxo_builder/utxo_coin_builder.rs @@ -5,9 +5,8 @@ use crate::utxo::rpc_clients::{ElectrumClient, ElectrumClientSettings, ElectrumC use crate::utxo::tx_cache::{UtxoVerboseCacheOps, UtxoVerboseCacheShared}; use crate::utxo::utxo_block_header_storage::BlockHeaderStorage; use crate::utxo::utxo_builder::utxo_conf_builder::{UtxoConfBuilder, UtxoConfError}; -use crate::utxo::{output_script, ElectrumBuilderArgs, RecentlySpentOutPoints, ScripthashNotificationSender, TxFee, - UtxoCoinConf, UtxoCoinFields, UtxoHDWallet, UtxoRpcMode, UtxoSyncStatus, UtxoSyncStatusLoopHandle, - UTXO_DUST_AMOUNT}; +use crate::utxo::{output_script, ElectrumBuilderArgs, RecentlySpentOutPoints, TxFee, UtxoCoinConf, UtxoCoinFields, + UtxoHDWallet, UtxoRpcMode, UtxoSyncStatus, UtxoSyncStatusLoopHandle, UTXO_DUST_AMOUNT}; use crate::{BlockchainNetwork, CoinTransportMetrics, DerivationMethod, HistorySyncState, IguanaPrivKey, PrivKeyBuildPolicy, PrivKeyPolicy, PrivKeyPolicyNotAllowed, RpcClientType, SharableRpcTransportEventHandler, UtxoActivationParams}; @@ -30,7 +29,6 @@ use serde_json::{self as json, Value as Json}; use spv_validation::conf::SPVConf; use spv_validation::helpers_validation::SPVError; use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps}; -use std::sync::Arc; use std::sync::Mutex; cfg_native! { @@ -38,6 +36,7 @@ cfg_native! { use crate::utxo::rpc_clients::{ConcurrentRequestMap, NativeClient, NativeClientImpl}; use dirs::home_dir; use std::path::{Path, PathBuf}; + use std::sync::Arc; } /// Number of seconds in a day (24 hours * 60 * 60) @@ -254,7 +253,7 @@ where // all spawned futures related to this `UTXO` coin will be aborted as well. let abortable_system: AbortableQueue = builder.ctx().abortable_system.create_subsystem()?; - let rpc_client = builder.rpc_client(None, abortable_system.create_subsystem()?).await?; + let rpc_client = builder.rpc_client(abortable_system.create_subsystem()?).await?; let tx_fee = builder.tx_fee(&rpc_client).await?; let decimals = builder.decimals(&rpc_client).await?; let dust_amount = builder.dust_amount(); @@ -332,7 +331,7 @@ pub trait UtxoFieldsWithHardwareWalletBuilder: UtxoCoinBuilderCommonOps { // all spawned futures related to this `UTXO` coin will be aborted as well. let abortable_system: AbortableQueue = self.ctx().abortable_system.create_subsystem()?; - let rpc_client = self.rpc_client(None, abortable_system.create_subsystem()?).await?; + let rpc_client = self.rpc_client(abortable_system.create_subsystem()?).await?; let tx_fee = self.tx_fee(&rpc_client).await?; let decimals = self.decimals(&rpc_client).await?; let dust_amount = self.dust_amount(); @@ -495,11 +494,7 @@ pub trait UtxoCoinBuilderCommonOps { } } - async fn rpc_client( - &self, - scripthash_notification_sender: ScripthashNotificationSender, - abortable_system: AbortableQueue, - ) -> UtxoCoinBuildResult { + async fn rpc_client(&self, abortable_system: AbortableQueue) -> UtxoCoinBuildResult { match self.activation_params().mode.clone() { UtxoRpcMode::Native => { #[cfg(target_arch = "wasm32")] @@ -523,7 +518,6 @@ pub trait UtxoCoinBuilderCommonOps { ElectrumBuilderArgs::default(), servers, (min_connected, max_connected), - scripthash_notification_sender, ) .await?; Ok(UtxoRpcClientEnum::Electrum(electrum)) @@ -539,7 +533,6 @@ pub trait UtxoCoinBuilderCommonOps { args: ElectrumBuilderArgs, servers: Vec, (min_connected, max_connected): (Option, Option), - scripthash_notification_sender: ScripthashNotificationSender, ) -> UtxoCoinBuildResult { let coin_ticker = self.ticker().to_owned(); let ctx = self.ctx(); @@ -576,8 +569,8 @@ pub trait UtxoCoinBuilderCommonOps { client_settings, event_handlers, block_headers_storage, + ctx.event_stream_manager.clone(), abortable_system, - scripthash_notification_sender, ) .map_to_mm(UtxoCoinBuildError::Internal) } diff --git a/mm2src/coins/utxo/utxo_tests.rs b/mm2src/coins/utxo/utxo_tests.rs index 702f269feb..0f376620db 100644 --- a/mm2src/coins/utxo/utxo_tests.rs +++ b/mm2src/coins/utxo/utxo_tests.rs @@ -42,6 +42,7 @@ use futures::channel::mpsc::channel; use futures::future::{join_all, Either, FutureExt, TryFutureExt}; use keys::prefixes::*; use mm2_core::mm_ctx::MmCtxBuilder; +use mm2_event_stream::StreamingManager; use mm2_number::bigdecimal::{BigDecimal, Signed}; use mm2_test_helpers::electrums::doc_electrums; use mm2_test_helpers::for_tests::{electrum_servers_rpc, mm_ctx_with_custom_db, DOC_ELECTRUM_ADDRS, @@ -85,7 +86,7 @@ pub fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { let servers = servers.into_iter().map(|s| json::from_value(s).unwrap()).collect(); let abortable_system = AbortableQueue::default(); - block_on(builder.electrum_client(abortable_system, args, servers, (None, None), None)).unwrap() + block_on(builder.electrum_client(abortable_system, args, servers, (None, None))).unwrap() } /// Returned client won't work by default, requires some mocks to be usable @@ -481,8 +482,8 @@ fn test_wait_for_payment_spend_timeout_electrum() { client_settings, Default::default(), block_headers_storage, + StreamingManager::default(), abortable_system, - None, ) .expect("Expected electrum_client_impl constructed without a problem"); let client = UtxoRpcClientEnum::Electrum(client); @@ -1518,13 +1519,13 @@ fn test_network_info_negative_time_offset() { #[test] fn test_unavailable_electrum_proto_version() { ElectrumClientImpl::try_new_arc.mock_safe( - |client_settings, block_headers_storage, abortable_system, event_handlers, scripthash_notification_sender| { + |client_settings, block_headers_storage, streaming_manager, abortable_system, event_handlers| { MockResult::Return(ElectrumClientImpl::with_protocol_version( client_settings, block_headers_storage, + streaming_manager, abortable_system, event_handlers, - scripthash_notification_sender, OrdRange::new(1.8, 1.9).unwrap(), )) }, @@ -1599,13 +1600,13 @@ fn test_spam_rick() { fn test_one_unavailable_electrum_proto_version() { // Patch the electurm client construct to require protocol version 1.4 only. ElectrumClientImpl::try_new_arc.mock_safe( - |client_settings, block_headers_storage, abortable_system, event_handlers, scripthash_notification_sender| { + |client_settings, block_headers_storage, streaming_manager, abortable_system, event_handlers| { MockResult::Return(ElectrumClientImpl::with_protocol_version( client_settings, block_headers_storage, + streaming_manager, abortable_system, event_handlers, - scripthash_notification_sender, OrdRange::new(1.4, 1.4).unwrap(), )) }, diff --git a/mm2src/coins/utxo/utxo_wasm_tests.rs b/mm2src/coins/utxo/utxo_wasm_tests.rs index bd059c8627..2241f40781 100644 --- a/mm2src/coins/utxo/utxo_wasm_tests.rs +++ b/mm2src/coins/utxo/utxo_wasm_tests.rs @@ -42,7 +42,7 @@ pub async fn electrum_client_for_test(servers: &[&str]) -> ElectrumClient { let servers = servers.into_iter().map(|s| json::from_value(s).unwrap()).collect(); let abortable_system = AbortableQueue::default(); builder - .electrum_client(abortable_system, args, servers, (None, None), None) + .electrum_client(abortable_system, args, servers, (None, None)) .await .unwrap() } diff --git a/mm2src/coins/z_coin/z_balance_streaming.rs b/mm2src/coins/z_coin/z_balance_streaming.rs index 18b2c6ef38..0760bfc929 100644 --- a/mm2src/coins/z_coin/z_balance_streaming.rs +++ b/mm2src/coins/z_coin/z_balance_streaming.rs @@ -4,15 +4,9 @@ use crate::MarketCoinOps; use async_trait::async_trait; use common::log::error; -use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::oneshot; -use futures::lock::Mutex as AsyncMutex; use futures_util::StreamExt; use mm2_event_stream::{Broadcaster, Event, EventStreamer, StreamHandlerInput}; -use std::sync::Arc; - -pub type ZBalanceEventSender = UnboundedSender<()>; -pub type ZBalanceEventHandler = Arc>>; pub struct ZCoinBalanceEventStreamer { coin: ZCoin, diff --git a/mm2src/coins/z_coin/z_coin_native_tests.rs b/mm2src/coins/z_coin/z_coin_native_tests.rs index de87c97b3c..a8436dc2d0 100644 --- a/mm2src/coins/z_coin/z_coin_native_tests.rs +++ b/mm2src/coins/z_coin/z_coin_native_tests.rs @@ -33,7 +33,7 @@ fn light_zcoin_activation_params() -> ZcoinActivationParams { url: s.to_string(), protocol: Default::default(), disable_cert_verification: Default::default(), - timeout_sec: None + timeout_sec: None, }) .collect(), min_connected: None, diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 5c49a6ca1f..1433dd719e 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -31,6 +31,7 @@ pub enum StreamingManagerError { ClientAlreadyListening, } +#[derive(Debug)] struct StreamerInfo { /// The communication channel to the streamer. data_in: Option>>, @@ -56,6 +57,7 @@ impl StreamerInfo { fn is_down(&self) -> bool { self.shutdown.is_canceled() } } +#[derive(Debug)] struct ClientInfo { /// The streamers the client is listening to. listening_to: HashSet, @@ -90,7 +92,7 @@ impl ClientInfo { } } -#[derive(Default)] +#[derive(Default, Debug)] struct StreamingManagerInner { /// A map from streamer IDs to their communication channels (if present) and shutdown handles. streamers: HashMap, @@ -98,7 +100,7 @@ struct StreamingManagerInner { clients: HashMap, } -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct StreamingManager(Arc>); impl StreamingManager { From c2f02442e2d5bf6b2de760b465d9ca7ffbd00e07 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 30 Oct 2024 22:35:13 +0100 Subject: [PATCH 119/128] fix: actually (auto) remove the client when they disconnect this commit makes it so that the client deletion logic is handled by the streaming manager when initializing the client. the return type of client initialization has an extra field that will run the client delection logic on Drop. this fixes the problem with clients never getting removed as when the clients disconnect and the body stream is no longer drived further thus will not hit the `remove_client` call. --- mm2src/common/common.rs | 1 + mm2src/common/on_drop_callback.rs | 19 ++++++++++++ mm2src/mm2_event_stream/src/manager.rs | 31 +++++++++++++++++-- .../src/event_streaming/sse_handler.rs | 2 -- .../src/event_streaming/wasm_event_stream.rs | 2 -- 5 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 mm2src/common/on_drop_callback.rs diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index de201856d8..54a9a37d40 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -131,6 +131,7 @@ pub mod custom_iter; pub mod expirable_map; pub mod notifier; pub mod number_type_casting; +pub mod on_drop_callback; pub mod password_policy; pub mod seri; pub mod time_cache; diff --git a/mm2src/common/on_drop_callback.rs b/mm2src/common/on_drop_callback.rs new file mode 100644 index 0000000000..a454cc9554 --- /dev/null +++ b/mm2src/common/on_drop_callback.rs @@ -0,0 +1,19 @@ +/// Runs some function when this object is dropped. +/// +/// We wrap the callback function in an `Option` so that we can exercise the less strict `FnOnce` bound +/// (`FnOnce` is less strict than `Fn`). This way we can take out the function and execute it when dropping. +/// We also implement this with `Box` instead of generics so not to force users to use generics if +/// this callback handle is stored in some struct. +pub struct OnDropCallback(Option>); + +impl OnDropCallback { + pub fn new(f: impl FnOnce() + Send + 'static) -> Self { Self(Some(Box::new(f))) } +} + +impl Drop for OnDropCallback { + fn drop(&mut self) { + if let Some(func) = self.0.take() { + func() + } + } +} diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 1433dd719e..ba2dc3c7b7 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -1,5 +1,6 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; +use std::ops::{Deref, DerefMut}; use std::sync::Arc; use crate::streamer::spawn; @@ -7,6 +8,7 @@ use crate::{Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; use common::log; +use common::on_drop_callback::OnDropCallback; use futures::channel::mpsc::UnboundedSender; use futures::channel::oneshot; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -263,7 +265,7 @@ impl StreamingManager { } /// Creates a new client and returns the event receiver for this client. - pub fn new_client(&self, client_id: u64) -> Result>, StreamingManagerError> { + pub fn new_client(&self, client_id: u64) -> Result { let mut this = self.write(); if this.clients.contains_key(&client_id) { return Err(StreamingManagerError::ClientExists); @@ -273,7 +275,13 @@ impl StreamingManager { let (tx, rx) = mpsc::channel(1024); let client_info = ClientInfo::new(tx); this.clients.insert(client_id, client_info); - Ok(rx) + let manager = self.clone(); + Ok(ClientHandle { + rx, + _on_drop_callback: OnDropCallback::new(move || { + manager.remove_client(client_id).ok(); + }), + }) } /// Removes a client from the manager. @@ -323,6 +331,25 @@ impl StreamingManager { } } +/// A handle that is returned on [`StreamingManager::new_client`] calls that will auto remove +/// the client when dropped. +/// So this handle must live as long as the client is connected. +pub struct ClientHandle { + rx: mpsc::Receiver>, + _on_drop_callback: OnDropCallback, +} + +/// Deref the handle to the receiver inside for ease of use. +impl Deref for ClientHandle { + type Target = mpsc::Receiver>; + fn deref(&self) -> &Self::Target { &self.rx } +} + +/// Also DerefMut since the receiver inside is mutated when consumed. +impl DerefMut for ClientHandle { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.rx } +} + #[cfg(test)] mod tests { use super::*; diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index 4785e121ea..534e32e614 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -43,8 +43,6 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Response { yield Ok::<_, hyper::Error>(Bytes::from(format!("data: {data} \n\n"))); } - // Inform the event stream manager that the client has disconnected. - event_stream_manager.remove_client(client_id).ok(); }); let response = Response::builder() diff --git a/mm2src/mm2_net/src/event_streaming/wasm_event_stream.rs b/mm2src/mm2_net/src/event_streaming/wasm_event_stream.rs index 102d808b8b..c10f838a70 100644 --- a/mm2src/mm2_net/src/event_streaming/wasm_event_stream.rs +++ b/mm2src/mm2_net/src/event_streaming/wasm_event_stream.rs @@ -43,6 +43,4 @@ pub async fn handle_worker_stream(ctx: MmArc, worker_path: String) { This could be due to the browser being incompatible.\n\ For more details, please refer to https://developer.mozilla.org/en-US/docs/Web/API/MessagePort/postMessage#browser_compatibility"); } - - event_stream_manager.remove_client(0).ok(); } From c29cf813aeecfbad0af8a2906fcf0c4c744fb796 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Thu, 31 Oct 2024 10:14:18 +0100 Subject: [PATCH 120/128] tests(streaming-manager): fix client handle dropped early --- mm2src/mm2_event_stream/src/manager.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index ba2dc3c7b7..c81e7e46e7 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -365,14 +365,16 @@ mod tests { let client_id2 = 2; let client_id3 = 3; - assert!(matches!(manager.new_client(client_id1), Ok(..))); + let c1_handle = manager.new_client(client_id1); + assert!(matches!(c1_handle, Ok(..))); // Adding the same client again should fail. assert!(matches!( manager.new_client(client_id1), Err(StreamingManagerError::ClientExists) )); // Adding a different new client should be OK. - assert!(matches!(manager.new_client(client_id2), Ok(..))); + let c2_handle = manager.new_client(client_id2); + assert!(matches!(c2_handle, Ok(..))); assert!(matches!(manager.remove_client(client_id1), Ok(()))); // Removing a removed client should fail. From 2a5f66c20cb290fc6ba04c07cc018f069cf32900 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 16 Aug 2024 19:53:25 +0200 Subject: [PATCH 121/128] z_coin wasm tx_history streaming test ignoring the non-wasm test for now since it needs zcash params downloaded which we don't have an auto way to do. would be better if we can download them in the same way we do for wasm (right from kdf). --- mm2src/coins/z_coin.rs | 8 +- .../coins/z_coin/storage/walletdb/wasm/mod.rs | 41 ++++----- mm2src/coins/z_coin/tx_streaming_tests/mod.rs | 30 +++++++ .../coins/z_coin/tx_streaming_tests/native.rs | 73 +++++++++++++++ .../coins/z_coin/tx_streaming_tests/wasm.rs | 74 +++++++++++++++ mm2src/coins/z_coin/z_coin_native_tests.rs | 90 +------------------ .../src/data/legacy/activation/utxo.rs | 12 +-- 7 files changed, 209 insertions(+), 119 deletions(-) create mode 100644 mm2src/coins/z_coin/tx_streaming_tests/mod.rs create mode 100644 mm2src/coins/z_coin/tx_streaming_tests/native.rs create mode 100644 mm2src/coins/z_coin/tx_streaming_tests/wasm.rs diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a703a0bc62..805d18d118 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,8 +1,10 @@ pub mod storage; pub mod tx_history_events; +#[cfg_attr(not(target_arch = "wasm32"), cfg(test))] +mod tx_streaming_tests; pub mod z_balance_streaming; mod z_coin_errors; -#[cfg(all(test, feature = "zhtlc-native-tests"))] +#[cfg(all(test, not(target_arch = "wasm32"), feature = "zhtlc-native-tests"))] mod z_coin_native_tests; mod z_htlc; mod z_rpc; @@ -780,8 +782,8 @@ impl Default for ZcoinActivationParams { requires_notarization: None, zcash_params_path: None, scan_blocks_per_iteration: NonZeroU32::new(1000).expect("1000 is a valid value"), - scan_interval_ms: 0, - account: 0, + scan_interval_ms: Default::default(), + account: Default::default(), } } } diff --git a/mm2src/coins/z_coin/storage/walletdb/wasm/mod.rs b/mm2src/coins/z_coin/storage/walletdb/wasm/mod.rs index 4c68fec22c..c1ffdfb0a2 100644 --- a/mm2src/coins/z_coin/storage/walletdb/wasm/mod.rs +++ b/mm2src/coins/z_coin/storage/walletdb/wasm/mod.rs @@ -71,6 +71,7 @@ mod wasm_test { use crate::z_coin::{ValidateBlocksError, ZcoinConsensusParams, ZcoinStorageError}; use crate::ZcoinProtocolInfo; use mm2_core::mm_ctx::MmArc; + use mm2_event_stream::StreamingManager; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use protobuf::Message; use std::path::PathBuf; @@ -255,7 +256,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -300,7 +301,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -359,7 +360,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -453,7 +454,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -542,7 +543,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -572,7 +573,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -611,7 +612,7 @@ mod wasm_test { blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -629,7 +630,7 @@ mod wasm_test { let scan = blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -653,7 +654,7 @@ mod wasm_test { assert!(blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None ) @@ -697,7 +698,7 @@ mod wasm_test { assert!(blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None ) @@ -718,7 +719,7 @@ mod wasm_test { assert!(blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None ) @@ -760,7 +761,7 @@ mod wasm_test { assert!(blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None ) @@ -790,7 +791,7 @@ mod wasm_test { let scan = blockdb .process_blocks_with_mode( consensus_params.clone(), - BlockProcessingMode::Scan(scan, None), + BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None, ) @@ -832,7 +833,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // assert!(blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .is_ok()); // @@ -852,7 +853,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // assert!(blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .is_ok()); // @@ -897,7 +898,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // assert!(blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .is_ok()); // @@ -928,7 +929,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // assert!(blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .is_ok()); // @@ -1098,7 +1099,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .unwrap(); // assert_eq!(walletdb.get_balance(AccountId(0)).await.unwrap(), value); @@ -1155,7 +1156,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .unwrap(); // @@ -1191,7 +1192,7 @@ mod wasm_test { // // Scan the cache // let scan = DataConnStmtCacheWrapper::new(DataConnStmtCacheWasm(walletdb.clone())); // blockdb - // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, None), None, None) + // .process_blocks_with_mode(consensus_params.clone(), BlockProcessingMode::Scan(scan, StreamingManager::default()), None, None) // .await // .unwrap(); // diff --git a/mm2src/coins/z_coin/tx_streaming_tests/mod.rs b/mm2src/coins/z_coin/tx_streaming_tests/mod.rs new file mode 100644 index 0000000000..457162c80f --- /dev/null +++ b/mm2src/coins/z_coin/tx_streaming_tests/mod.rs @@ -0,0 +1,30 @@ +#[cfg(not(target_arch = "wasm32"))] mod native; +#[cfg(target_arch = "wasm32")] mod wasm; + +use common::now_sec; +use mm2_test_helpers::for_tests::{PIRATE_ELECTRUMS, PIRATE_LIGHTWALLETD_URLS}; + +use crate::utxo::rpc_clients::ElectrumConnectionSettings; +use crate::z_coin::{ZcoinActivationParams, ZcoinRpcMode}; + +fn light_zcoin_activation_params() -> ZcoinActivationParams { + ZcoinActivationParams { + mode: ZcoinRpcMode::Light { + electrum_servers: PIRATE_ELECTRUMS + .iter() + .map(|s| ElectrumConnectionSettings { + url: s.to_string(), + protocol: Default::default(), + disable_cert_verification: Default::default(), + timeout_sec: None, + }) + .collect(), + min_connected: None, + max_connected: None, + light_wallet_d_servers: PIRATE_LIGHTWALLETD_URLS.iter().map(|s| s.to_string()).collect(), + sync_params: Some(crate::z_coin::SyncStartPoint::Date(now_sec() - 24 * 60 * 60)), + skip_sync_params: None, + }, + ..Default::default() + } +} diff --git a/mm2src/coins/z_coin/tx_streaming_tests/native.rs b/mm2src/coins/z_coin/tx_streaming_tests/native.rs new file mode 100644 index 0000000000..cc5ecc5812 --- /dev/null +++ b/mm2src/coins/z_coin/tx_streaming_tests/native.rs @@ -0,0 +1,73 @@ +use common::custom_futures::timeout::FutureTimerExt; +use common::{block_on, Future01CompatExt}; +use mm2_core::mm_ctx::MmCtxBuilder; +use mm2_test_helpers::for_tests::{pirate_conf, ARRR}; +use std::time::Duration; + +use super::light_zcoin_activation_params; +use crate::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; +use crate::z_coin::z_coin_from_conf_and_params; +use crate::z_coin::z_htlc::z_send_dex_fee; +use crate::{CoinProtocol, MarketCoinOps, MmCoin, PrivKeyBuildPolicy}; + +#[test] +#[ignore] // Ignored because we don't have zcash params in CI. TODO: Why not download them on demand like how we do in wasm (see download_and_save_params). +fn test_zcoin_tx_streaming() { + let ctx = MmCtxBuilder::default().into_mm_arc(); + let conf = pirate_conf(); + let params = light_zcoin_activation_params(); + // Address: RQX5MnqnxEk6P33LSEAxC2vqA7DfSdWVyH + // Or: zs1n2azlwcj9pvl2eh36qvzgeukt2cpzmw44hya8wyu52j663d0dfs4d5hjx6tr04trz34jxyy433j + let priv_key_policy = + PrivKeyBuildPolicy::IguanaPrivKey("6d862798ef956fb60fb17bcc417dd6d44bfff066a4a49301cd2528e41a4a3e45".into()); + let protocol_info = match serde_json::from_value::(conf["protocol"].clone()).unwrap() { + CoinProtocol::ZHTLC(protocol_info) => protocol_info, + other_protocol => panic!("Failed to get protocol from config: {:?}", other_protocol), + }; + + let coin = block_on(z_coin_from_conf_and_params( + &ctx, + ARRR, + &conf, + ¶ms, + protocol_info, + priv_key_policy, + )) + .unwrap(); + + // Wait till we are synced with the sapling state. + while !block_on(coin.is_sapling_state_synced()) { + std::thread::sleep(Duration::from_secs(1)); + } + + // Query the block height to make sure our electrums are actually connected. + log!("current block = {:?}", block_on(coin.current_block().compat()).unwrap()); + + // Add a new client to use it for listening to tx history events. + let client_id = 1; + let mut event_receiver = ctx.event_stream_manager.new_client(client_id).unwrap(); + // Add the streamer that will stream the tx history events. + let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); + // Subscribe the client to the streamer. + block_on(ctx.event_stream_manager.add(client_id, streamer, coin.spawner())).unwrap(); + + // Send a tx to have it in the tx history. + let tx = block_on(z_send_dex_fee(&coin, "0.0001".parse().unwrap(), &[1; 16])).unwrap(); + + // Wait for the tx history event (should be streamed next block). + let event = block_on(Box::pin(event_receiver.recv()).timeout_secs(120.)) + .expect("timed out waiting for tx to showup") + .expect("tx history sender shutdown"); + + log!("{:?}", event.get()); + let (event_type, event_data) = event.get(); + // Make sure this is not an error event, + assert!(!event_type.starts_with("ERROR_")); + // from the expected streamer, + assert_eq!( + event_type, + ZCoinTxHistoryEventStreamer::derive_streamer_id(coin.ticker()) + ); + // and has the expected data. + assert_eq!(event_data["tx_hash"].as_str().unwrap(), tx.txid().to_string()); +} diff --git a/mm2src/coins/z_coin/tx_streaming_tests/wasm.rs b/mm2src/coins/z_coin/tx_streaming_tests/wasm.rs new file mode 100644 index 0000000000..192db3c7a9 --- /dev/null +++ b/mm2src/coins/z_coin/tx_streaming_tests/wasm.rs @@ -0,0 +1,74 @@ +use common::custom_futures::timeout::FutureTimerExt; +use common::{executor::Timer, Future01CompatExt}; +use mm2_core::mm_ctx::MmCtxBuilder; +use mm2_test_helpers::for_tests::{pirate_conf, ARRR}; +use wasm_bindgen_test::*; + +use super::light_zcoin_activation_params; +use crate::z_coin::tx_history_events::ZCoinTxHistoryEventStreamer; +use crate::z_coin::z_coin_from_conf_and_params; +use crate::z_coin::z_htlc::z_send_dex_fee; +use crate::PrivKeyBuildPolicy; +use crate::{CoinProtocol, MarketCoinOps, MmCoin}; + +#[wasm_bindgen_test] +async fn test_zcoin_tx_streaming() { + let ctx = MmCtxBuilder::default().into_mm_arc(); + let conf = pirate_conf(); + let params = light_zcoin_activation_params(); + // Address: RQX5MnqnxEk6P33LSEAxC2vqA7DfSdWVyH + // Or: zs1n2azlwcj9pvl2eh36qvzgeukt2cpzmw44hya8wyu52j663d0dfs4d5hjx6tr04trz34jxyy433j + let priv_key_policy = + PrivKeyBuildPolicy::IguanaPrivKey("6d862798ef956fb60fb17bcc417dd6d44bfff066a4a49301cd2528e41a4a3e45".into()); + let protocol_info = match serde_json::from_value::(conf["protocol"].clone()).unwrap() { + CoinProtocol::ZHTLC(protocol_info) => protocol_info, + other_protocol => panic!("Failed to get protocol from config: {:?}", other_protocol), + }; + + let coin = z_coin_from_conf_and_params(&ctx, ARRR, &conf, ¶ms, protocol_info, priv_key_policy) + .await + .unwrap(); + + // Wait till we are synced with the sapling state. + while !coin.is_sapling_state_synced().await { + Timer::sleep(1.).await; + } + + // Query the block height to make sure our electrums are actually connected. + log!("current block = {:?}", coin.current_block().compat().await.unwrap()); + + // Add a new client to use it for listening to tx history events. + let client_id = 1; + let mut event_receiver = ctx.event_stream_manager.new_client(client_id).unwrap(); + // Add the streamer that will stream the tx history events. + let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); + // Subscribe the client to the streamer. + ctx.event_stream_manager + .add(client_id, streamer, coin.spawner()) + .await + .unwrap(); + + // Send a tx to have it in the tx history. + let tx = z_send_dex_fee(&coin, "0.0001".parse().unwrap(), &[1; 16]) + .await + .unwrap(); + + // Wait for the tx history event (should be streamed next block). + let event = Box::pin(event_receiver.recv()) + .timeout_secs(120.) + .await + .expect("timed out waiting for tx to showup") + .expect("tx history sender shutdown"); + + log!("{:?}", event.get()); + let (event_type, event_data) = event.get(); + // Make sure this is not an error event, + assert!(!event_type.starts_with("ERROR_")); + // from the expected streamer, + assert_eq!( + event_type, + ZCoinTxHistoryEventStreamer::derive_streamer_id(coin.ticker()) + ); + // and has the expected data. + assert_eq!(event_data["tx_hash"].as_str().unwrap(), tx.txid().to_string()); +} diff --git a/mm2src/coins/z_coin/z_coin_native_tests.rs b/mm2src/coins/z_coin/z_coin_native_tests.rs index a8436dc2d0..0cde681ee8 100644 --- a/mm2src/coins/z_coin/z_coin_native_tests.rs +++ b/mm2src/coins/z_coin/z_coin_native_tests.rs @@ -1,20 +1,16 @@ use bitcrypto::dhash160; -use common::custom_futures::timeout::FutureTimerExt; use common::{block_on, now_sec}; use mm2_core::mm_ctx::MmCtxBuilder; -use mm2_test_helpers::for_tests::{zombie_conf, ZOMBIE_ELECTRUMS, ZOMBIE_LIGHTWALLETD_URLS, ZOMBIE_TICKER}; +use mm2_test_helpers::for_tests::zombie_conf; use std::path::PathBuf; use std::time::Duration; use zcash_client_backend::encoding::decode_extended_spending_key; -use super::tx_history_events::ZCoinTxHistoryEventStreamer; use super::{z_coin_from_conf_and_params_with_z_key, z_mainnet_constants, PrivKeyBuildPolicy, RefundPaymentArgs, SendPaymentArgs, SpendPaymentArgs, SwapOps, ValidateFeeArgs, ValidatePaymentError, ZTransaction}; -use crate::utxo::rpc_clients::ElectrumConnectionSettings; -use crate::z_coin::z_coin_from_conf_and_params; use crate::z_coin::{z_htlc::z_send_dex_fee, ZcoinActivationParams, ZcoinRpcMode}; -use crate::{CoinProtocol, MarketCoinOps, SwapTxTypeWithSecretHash}; -use crate::{DexFee, MmCoin}; +use crate::DexFee; +use crate::{CoinProtocol, SwapTxTypeWithSecretHash}; use mm2_number::MmNumber; fn native_zcoin_activation_params() -> ZcoinActivationParams { @@ -24,28 +20,6 @@ fn native_zcoin_activation_params() -> ZcoinActivationParams { } } -fn light_zcoin_activation_params() -> ZcoinActivationParams { - ZcoinActivationParams { - mode: ZcoinRpcMode::Light { - electrum_servers: ZOMBIE_ELECTRUMS - .iter() - .map(|s| ElectrumConnectionSettings { - url: s.to_string(), - protocol: Default::default(), - disable_cert_verification: Default::default(), - timeout_sec: None, - }) - .collect(), - min_connected: None, - max_connected: None, - light_wallet_d_servers: ZOMBIE_LIGHTWALLETD_URLS.iter().map(|s| s.to_string()).collect(), - sync_params: Some(crate::z_coin::SyncStartPoint::Date(now_sec() - 24 * 60 * 60)), - skip_sync_params: None, - }, - ..Default::default() - } -} - #[test] fn zombie_coin_send_and_refund_maker_payment() { let ctx = MmCtxBuilder::default().into_mm_arc(); @@ -313,61 +287,3 @@ fn zombie_coin_validate_dex_fee() { }; block_on(coin.validate_fee(validate_fee_args)).unwrap(); } - -#[test] -fn test_zombie_tx_streaming() { - let ctx = MmCtxBuilder::default().into_mm_arc(); - let conf = zombie_conf(); - let params = light_zcoin_activation_params(); - // Address: RQX5MnqnxEk6P33LSEAxC2vqA7DfSdWVyH - // Or: zs1n2azlwcj9pvl2eh36qvzgeukt2cpzmw44hya8wyu52j663d0dfs4d5hjx6tr04trz34jxyy433j - let priv_key_policy = - PrivKeyBuildPolicy::IguanaPrivKey("6d862798ef956fb60fb17bcc417dd6d44bfff066a4a49301cd2528e41a4a3e45".into()); - let protocol_info = match serde_json::from_value::(conf["protocol"].clone()).unwrap() { - CoinProtocol::ZHTLC(protocol_info) => protocol_info, - other_protocol => panic!("Failed to get protocol from config: {:?}", other_protocol), - }; - - let coin = block_on(z_coin_from_conf_and_params( - &ctx, - ZOMBIE_TICKER, - &conf, - ¶ms, - protocol_info, - priv_key_policy, - )) - .unwrap(); - - // Wait till we are synced with the sapling state. - while !block_on(coin.is_sapling_state_synced()) { - std::thread::sleep(Duration::from_secs(1)); - } - - // Add a new client to use it for listening to tx history events. - let client_id = 1; - let mut event_receiver = ctx.event_stream_manager.new_client(client_id).unwrap(); - // Add the streamer that will stream the tx history events. - let streamer = ZCoinTxHistoryEventStreamer::new(coin.clone()); - // Subscribe the client to the streamer. - block_on(ctx.event_stream_manager.add(client_id, streamer, coin.spawner())).unwrap(); - - // Send a tx to have it in the tx history. - let tx = block_on(z_send_dex_fee(&coin, "0.0001".parse().unwrap(), &[1; 16])).unwrap(); - - // Wait for the tx history event (should be streamed next block). - let event = block_on(Box::pin(event_receiver.recv()).timeout_secs(120.)) - .expect("timed out waiting for tx to showup") - .expect("tx history sender shutdown"); - - log!("{:?}", event.get()); - let (event_type, event_data) = event.get(); - // Make sure this is not an error event, - assert!(!event_type.starts_with("ERROR_")); - // from the expected streamer, - assert_eq!( - event_type, - ZCoinTxHistoryEventStreamer::derive_streamer_id(coin.ticker()) - ); - // and has the expected data. - assert_eq!(event_data["tx_hash"].as_str().unwrap(), tx.txid().to_string()); -} diff --git a/mm2src/mm2_rpc/src/data/legacy/activation/utxo.rs b/mm2src/mm2_rpc/src/data/legacy/activation/utxo.rs index d585424c57..a14f5edc13 100644 --- a/mm2src/mm2_rpc/src/data/legacy/activation/utxo.rs +++ b/mm2src/mm2_rpc/src/data/legacy/activation/utxo.rs @@ -11,23 +11,17 @@ pub struct UtxoMergeParams { } #[allow(clippy::upper_case_acronyms)] -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] /// Deserializable Electrum protocol representation for RPC -#[derive(Default)] pub enum ElectrumProtocol { /// TCP - #[default] + #[cfg_attr(not(target_arch = "wasm32"), default)] TCP, /// SSL/TLS SSL, /// Insecure WebSocket. WS, /// Secure WebSocket. + #[cfg_attr(target_arch = "wasm32", default)] WSS, } - -#[cfg(not(target_arch = "wasm32"))] -#[cfg(target_arch = "wasm32")] -impl Default for ElectrumProtocol { - fn default() -> Self { ElectrumProtocol::WS } -} From 159445b2da31c5bea76dfe3490759250f0dc6d19 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Fri, 1 Nov 2024 17:11:27 +0100 Subject: [PATCH 122/128] fix activate_z_coin_* naming was used loosely, pirate was used as if it was zombie, a previous fix in the url constants led to this breaking. --- .../mm2_main/tests/mm2_tests/z_coin_tests.rs | 42 +++++++++---------- mm2src/mm2_test_helpers/src/for_tests.rs | 4 +- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/mm2src/mm2_main/tests/mm2_tests/z_coin_tests.rs b/mm2src/mm2_main/tests/mm2_tests/z_coin_tests.rs index 004ee27cac..d99290c3dc 100644 --- a/mm2src/mm2_main/tests/mm2_tests/z_coin_tests.rs +++ b/mm2src/mm2_main/tests/mm2_tests/z_coin_tests.rs @@ -17,8 +17,8 @@ use std::str::FromStr; use std::thread; use std::time::Duration; -const ZOMBIE_TEST_BIP39_ACTIVATION_SEED: &str = "course flock lucky cereal hamster novel team never metal bean behind cute cruel matrix symptom fault harsh fashion impact prison glove then tree chef"; -const ZOMBIE_TEST_BALANCE_SEED: &str = "zombie test seed"; +const ARRR_TEST_BIP39_ACTIVATION_SEED: &str = "course flock lucky cereal hamster novel team never metal bean behind cute cruel matrix symptom fault harsh fashion impact prison glove then tree chef"; +const ARRR_TEST_BALANCE_SEED: &str = "zombie test seed"; const ARRR_TEST_ACTIVATION_SEED: &str = "arrr test activation seed"; const ZOMBIE_TEST_HISTORY_SEED: &str = "zombie test history seed"; const ZOMBIE_TEST_WITHDRAW_SEED: &str = "zombie withdraw test seed"; @@ -48,16 +48,16 @@ async fn withdraw(mm: &MarketMakerIt, coin: &str, to: &str, amount: &str) -> Tra #[test] fn activate_z_coin_light() { - let coins = json!([zombie_conf()]); + let coins = json!([pirate_conf()]); - let conf = Mm2TestConf::seednode(ZOMBIE_TEST_BALANCE_SEED, &coins); + let conf = Mm2TestConf::seednode(ARRR_TEST_BALANCE_SEED, &coins); let mm = MarketMakerIt::start(conf.conf, conf.rpc_password, None).unwrap(); let activation_result = block_on(enable_z_coin_light( &mm, - ZOMBIE_TICKER, - ZOMBIE_ELECTRUMS, - ZOMBIE_LIGHTWALLETD_URLS, + ARRR, + PIRATE_ELECTRUMS, + PIRATE_LIGHTWALLETD_URLS, None, None, )); @@ -71,16 +71,16 @@ fn activate_z_coin_light() { #[test] fn activate_z_coin_light_with_changing_height() { - let coins = json!([zombie_conf()]); + let coins = json!([pirate_conf()]); - let conf = Mm2TestConf::seednode_with_hd_account(ZOMBIE_TEST_BIP39_ACTIVATION_SEED, &coins); + let conf = Mm2TestConf::seednode_with_hd_account(ARRR_TEST_BIP39_ACTIVATION_SEED, &coins); let mm = MarketMakerIt::start(conf.conf, conf.rpc_password, None).unwrap(); let activation_result = block_on(enable_z_coin_light( &mm, - ZOMBIE_TICKER, - ZOMBIE_ELECTRUMS, - ZOMBIE_LIGHTWALLETD_URLS, + ARRR, + PIRATE_ELECTRUMS, + PIRATE_LIGHTWALLETD_URLS, None, None, )); @@ -93,7 +93,7 @@ fn activate_z_coin_light_with_changing_height() { assert_eq!(balance.balance.spendable, BigDecimal::default()); // disable coin - block_on(disable_coin(&mm, ZOMBIE_TICKER, true)); + block_on(disable_coin(&mm, ARRR, true)); // Perform activation with changed height // Calculate timestamp for 2 days ago @@ -106,9 +106,9 @@ fn activate_z_coin_light_with_changing_height() { let activation_result = block_on(enable_z_coin_light( &mm, - ZOMBIE_TICKER, - ZOMBIE_ELECTRUMS, - ZOMBIE_LIGHTWALLETD_URLS, + ARRR, + PIRATE_ELECTRUMS, + PIRATE_LIGHTWALLETD_URLS, None, Some(two_days_ago), )); @@ -132,17 +132,17 @@ fn activate_z_coin_light_with_changing_height() { #[test] fn activate_z_coin_with_hd_account() { - let coins = json!([zombie_conf()]); + let coins = json!([pirate_conf()]); let hd_account_id = 0; - let conf = Mm2TestConf::seednode_with_hd_account(ZOMBIE_TEST_BIP39_ACTIVATION_SEED, &coins); + let conf = Mm2TestConf::seednode_with_hd_account(ARRR_TEST_BIP39_ACTIVATION_SEED, &coins); let mm = MarketMakerIt::start(conf.conf, conf.rpc_password, None).unwrap(); let activation_result = block_on(enable_z_coin_light( &mm, - ZOMBIE_TICKER, - ZOMBIE_ELECTRUMS, - ZOMBIE_LIGHTWALLETD_URLS, + ARRR, + PIRATE_ELECTRUMS, + PIRATE_LIGHTWALLETD_URLS, Some(hd_account_id), None, )); diff --git a/mm2src/mm2_test_helpers/src/for_tests.rs b/mm2src/mm2_test_helpers/src/for_tests.rs index 003dd7f299..de26d80b44 100644 --- a/mm2src/mm2_test_helpers/src/for_tests.rs +++ b/mm2src/mm2_test_helpers/src/for_tests.rs @@ -523,9 +523,11 @@ pub fn pirate_conf() -> Json { "b58_pubkey_address_prefix": [ 28, 184 ], "b58_script_address_prefix": [ 28, 189 ] }, + "z_derivation_path": "m/32'/133'", } }, - "required_confirmations":0 + "required_confirmations":0, + "derivation_path": "m/44'/133'", }) } From 853bbf56fde086ab8d71dea2100e5c3567731673 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 2 Nov 2024 22:09:41 +0100 Subject: [PATCH 123/128] add a fixme; check me out --- mm2src/coins/eth/fee_estimation/eth_fee_events.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs index 8d91790374..8f49ebf72b 100644 --- a/mm2src/coins/eth/fee_estimation/eth_fee_events.rs +++ b/mm2src/coins/eth/fee_estimation/eth_fee_events.rs @@ -31,6 +31,7 @@ pub struct EthFeeStreamingConfig { impl Default for EthFeeStreamingConfig { fn default() -> Self { Self { + // FIXME: https://github.com/KomodoPlatform/komodo-defi-framework/pull/2172#discussion_r1785054117 estimate_every: 15.0, estimator_type: EstimatorType::Simple, } From 78cece1f1fd0f28ed5f579acaed4b9151eec0c90 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Sat, 2 Nov 2024 22:31:58 +0100 Subject: [PATCH 124/128] review(onur): cross test event streaming crate --- mm2src/mm2_event_stream/src/event.rs | 2 +- mm2src/mm2_event_stream/src/manager.rs | 37 ++++++++++++------------- mm2src/mm2_event_stream/src/streamer.rs | 24 ++++++++-------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index 57931b9817..ae66456eea 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -3,7 +3,7 @@ use serde_json::Value as Json; // Note `Event` shouldn't be `Clone`able, but rather Arc/Rc wrapped and then shared. // This is only for testing. /// Multi-purpose/generic event type that can easily be used over the event streaming -#[cfg_attr(test, derive(Clone, Debug, PartialEq))] +#[cfg_attr(any(test, target_arch = "wasm32"), derive(Clone, Debug, PartialEq))] #[derive(Default)] pub struct Event { /// The type of the event (balance, network, swap, etc...). diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index c81e7e46e7..b1d8e016f5 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -350,16 +350,20 @@ impl DerefMut for ClientHandle { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.rx } } -#[cfg(test)] +#[cfg(any(test, target_arch = "wasm32"))] mod tests { use super::*; use crate::streamer::test_utils::{InitErrorStreamer, PeriodicStreamer, ReactiveStreamer}; use common::executor::{abortable_queue::AbortableQueue, AbortableSystem, Timer}; + use common::{cfg_wasm32, cross_test}; use serde_json::json; + cfg_wasm32! { + use wasm_bindgen_test::*; + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + } - #[tokio::test] - async fn test_add_remove_client() { + cross_test!(test_add_remove_client, { let manager = StreamingManager::default(); let client_id1 = 1; let client_id2 = 2; @@ -387,10 +391,9 @@ mod tests { manager.remove_client(client_id3), Err(StreamingManagerError::UnknownClient) )); - } + }); - #[tokio::test] - async fn test_broadcast_all() { + cross_test!(test_broadcast_all, { // Create a manager and add register two clients with it. let manager = StreamingManager::default(); let mut client1 = manager.new_client(1).unwrap(); @@ -411,10 +414,9 @@ mod tests { // `recv` shouldn't work at this point since the client is removed. assert!(client1.try_recv().is_err()); assert!(client2.try_recv().is_err()); - } + }); - #[tokio::test] - async fn test_periodic_streamer() { + cross_test!(test_periodic_streamer, { let manager = StreamingManager::default(); let system = AbortableQueue::default(); let (client_id1, client_id2) = (1, 2); @@ -438,10 +440,9 @@ mod tests { // The other client shouldn't have received any events. assert!(client2.try_recv().is_err()); - } + }); - #[tokio::test] - async fn test_reactive_streamer() { + cross_test!(test_reactive_streamer, { let manager = StreamingManager::default(); let system = AbortableQueue::default(); let (client_id1, client_id2) = (1, 2); @@ -475,10 +476,9 @@ mod tests { // The other client shouldn't have received any events. assert!(client2.try_recv().is_err()); - } + }); - #[tokio::test] - async fn test_erroring_streamer() { + cross_test!(test_erroring_streamer, { let manager = StreamingManager::default(); let system = AbortableQueue::default(); let client_id = 1; @@ -491,10 +491,9 @@ mod tests { .unwrap_err(); assert!(matches!(error, StreamingManagerError::SpawnError(..))); - } + }); - #[tokio::test] - async fn test_remove_streamer_if_down() { + cross_test!(test_remove_streamer_if_down, { let manager = StreamingManager::default(); let system = AbortableQueue::default(); let client_id = 1; @@ -542,5 +541,5 @@ mod tests { .get(&client_id) .unwrap() .listens_to(&streamer_id)); - } + }); } diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index 6055d05149..cf63ab766f 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -113,7 +113,7 @@ impl Broadcaster { pub fn broadcast(&self, event: Event) { self.0.broadcast(event); } } -#[cfg(test)] +#[cfg(any(test, target_arch = "wasm32"))] pub mod test_utils { use super::*; @@ -188,15 +188,19 @@ pub mod test_utils { } } -#[cfg(test)] +#[cfg(any(test, target_arch = "wasm32"))] mod tests { use super::test_utils::{InitErrorStreamer, PeriodicStreamer, ReactiveStreamer}; use super::*; use common::executor::abortable_queue::AbortableQueue; + use common::{cfg_wasm32, cross_test}; + cfg_wasm32! { + use wasm_bindgen_test::*; + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + } - #[tokio::test] - async fn test_spawn_periodic_streamer() { + cross_test!(test_spawn_periodic_streamer, { let system = AbortableQueue::default(); // Spawn the periodic streamer. let (_, data_in) = spawn(PeriodicStreamer, system.weak_spawner(), StreamingManager::default()) @@ -204,10 +208,9 @@ mod tests { .unwrap(); // Periodic streamer shouldn't be ingesting any input. assert!(data_in.is_none()); - } + }); - #[tokio::test] - async fn test_spawn_reactive_streamer() { + cross_test!(test_spawn_reactive_streamer, { let system = AbortableQueue::default(); // Spawn the reactive streamer. let (_, data_in) = spawn(ReactiveStreamer, system.weak_spawner(), StreamingManager::default()) @@ -215,10 +218,9 @@ mod tests { .unwrap(); // Reactive streamer should be ingesting some input. assert!(data_in.is_some()); - } + }); - #[tokio::test] - async fn test_spawn_erroring_streamer() { + cross_test!(test_spawn_erroring_streamer, { let system = AbortableQueue::default(); // Try to spawn the erroring streamer. let err = spawn(InitErrorStreamer, system.weak_spawner(), StreamingManager::default()) @@ -226,5 +228,5 @@ mod tests { .unwrap_err(); // The streamer should return an error. assert_eq!(err, "error"); - } + }); } From 65bd66253220e210d3b85354deb0ae6588010020 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 6 Nov 2024 10:42:18 +0100 Subject: [PATCH 125/128] review(onur): don't expose ctx from MmCoin pass the streaming manager down to where it should be used instead this does so for utxo balance history, tendermint to come --- mm2src/coins/eth.rs | 2 -- mm2src/coins/lightning.rs | 2 -- mm2src/coins/lp_coins.rs | 6 ------ mm2src/coins/qrc20.rs | 2 -- mm2src/coins/siacoin.rs | 2 -- mm2src/coins/tendermint/tendermint_coin.rs | 2 -- mm2src/coins/tendermint/tendermint_token.rs | 2 -- mm2src/coins/tendermint/tendermint_tx_history_v2.rs | 3 ++- mm2src/coins/test_coin.rs | 2 -- mm2src/coins/utxo/bch.rs | 2 -- mm2src/coins/utxo/qtum.rs | 2 -- mm2src/coins/utxo/slp.rs | 2 -- mm2src/coins/utxo/utxo_common_tests.rs | 2 ++ mm2src/coins/utxo/utxo_standard.rs | 2 -- mm2src/coins/utxo/utxo_tx_history_v2.rs | 10 ++++++++-- mm2src/coins/z_coin.rs | 2 -- .../coins_activation/src/bch_with_tokens_activation.rs | 8 +++++++- mm2src/coins_activation/src/sia_coin_activation.rs | 3 +++ .../src/standalone_coin/init_standalone_coin.rs | 3 +++ .../src/utxo_activation/common_impl.rs | 4 +++- .../src/utxo_activation/init_bch_activation.rs | 4 +++- .../src/utxo_activation/init_qtum_activation.rs | 4 +++- .../utxo_activation/init_utxo_standard_activation.rs | 4 +++- mm2src/coins_activation/src/z_coin_activation.rs | 2 ++ 24 files changed, 39 insertions(+), 38 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 38c2e0f4eb..aa02924ab9 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -5447,8 +5447,6 @@ impl EthTxFeeDetails { impl MmCoin for EthCoin { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.ctx) } - fn spawner(&self) -> WeakSpawner { self.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/lightning.rs b/mm2src/coins/lightning.rs index 754a75a000..af2354edf0 100644 --- a/mm2src/coins/lightning.rs +++ b/mm2src/coins/lightning.rs @@ -1267,8 +1267,6 @@ struct LightningProtocolInfo { impl MmCoin for LightningCoin { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { self.platform.coin.get_ctx() } - fn spawner(&self) -> WeakSpawner { self.platform.abortable_system.weak_spawner() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 7a676d446e..83509cada7 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -3271,12 +3271,6 @@ pub trait MmCoin: coin_conf["wallet_only"].as_bool().unwrap_or(false) } - /// Returns the MmArc context the coin is running on top of. - /// - /// This should really never fail (return `None`), but it's here since coins hold a weak - /// reference to the context (TODO: why?). - fn get_ctx(&self) -> Option; - /// Returns a spawner pinned to the coin. /// /// # Note diff --git a/mm2src/coins/qrc20.rs b/mm2src/coins/qrc20.rs index 5498f9574a..7bfa092ab5 100644 --- a/mm2src/coins/qrc20.rs +++ b/mm2src/coins/qrc20.rs @@ -1300,8 +1300,6 @@ impl MarketCoinOps for Qrc20Coin { impl MmCoin for Qrc20Coin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo) } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins/siacoin.rs b/mm2src/coins/siacoin.rs index d6be781762..a7f6ba5fa8 100644 --- a/mm2src/coins/siacoin.rs +++ b/mm2src/coins/siacoin.rs @@ -212,8 +212,6 @@ pub struct SiaCoinProtocolInfo; impl MmCoin for SiaCoin { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { unimplemented!() } - fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/tendermint/tendermint_coin.rs b/mm2src/coins/tendermint/tendermint_coin.rs index c6d6859e15..323c9da755 100644 --- a/mm2src/coins/tendermint/tendermint_coin.rs +++ b/mm2src/coins/tendermint/tendermint_coin.rs @@ -2160,8 +2160,6 @@ pub async fn get_ibc_chain_list() -> IBCChainRegistriesResult { impl MmCoin for TendermintCoin { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.ctx) } - fn wallet_only(&self, ctx: &MmArc) -> bool { let coin_conf = crate::coin_conf(ctx, self.ticker()); let wallet_only_conf = coin_conf["wallet_only"].as_bool().unwrap_or(false); diff --git a/mm2src/coins/tendermint/tendermint_token.rs b/mm2src/coins/tendermint/tendermint_token.rs index d02010cb40..1bb63dd0eb 100644 --- a/mm2src/coins/tendermint/tendermint_token.rs +++ b/mm2src/coins/tendermint/tendermint_token.rs @@ -488,8 +488,6 @@ impl MarketCoinOps for TendermintToken { impl MmCoin for TendermintToken { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { self.platform_coin.get_ctx() } - fn wallet_only(&self, ctx: &MmArc) -> bool { let coin_conf = crate::coin_conf(ctx, self.ticker()); let wallet_only_conf = coin_conf["wallet_only"].as_bool().unwrap_or(false); diff --git a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs index 9e83f707cc..6a06d000f4 100644 --- a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs +++ b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs @@ -18,6 +18,7 @@ use cosmrs::tendermint::abci::{Code as TxCode, EventAttribute}; use cosmrs::tx::Fee; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::MmResult; +use mm2_event_stream::StreamingManager; use mm2_number::BigDecimal; use mm2_state_machine::prelude::*; use mm2_state_machine::state_machine::StateMachineTrait; @@ -588,7 +589,7 @@ where "could not get rpc client" ); - let streaming_manager = coin.get_ctx().unwrap().event_stream_manager.clone(); + let streaming_manager: StreamingManager = panic!(); loop { let response = try_or_return_stopped_as_err!( client diff --git a/mm2src/coins/test_coin.rs b/mm2src/coins/test_coin.rs index 2dec95f7b2..0669d7e014 100644 --- a/mm2src/coins/test_coin.rs +++ b/mm2src/coins/test_coin.rs @@ -343,8 +343,6 @@ impl WatcherOps for TestCoin { impl MmCoin for TestCoin { fn is_asset_chain(&self) -> bool { unimplemented!() } - fn get_ctx(&self) -> Option { unimplemented!() } - fn spawner(&self) -> WeakSpawner { unimplemented!() } fn get_raw_transaction(&self, _req: RawTransactionRequest) -> RawTransactionFut { unimplemented!() } diff --git a/mm2src/coins/utxo/bch.rs b/mm2src/coins/utxo/bch.rs index b700b767e5..ec61314ec6 100644 --- a/mm2src/coins/utxo/bch.rs +++ b/mm2src/coins/utxo/bch.rs @@ -1288,8 +1288,6 @@ impl MarketCoinOps for BchCoin { impl MmCoin for BchCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/qtum.rs b/mm2src/coins/utxo/qtum.rs index 34bc309acd..e7d9455cde 100644 --- a/mm2src/coins/utxo/qtum.rs +++ b/mm2src/coins/utxo/qtum.rs @@ -903,8 +903,6 @@ impl MarketCoinOps for QtumCoin { impl MmCoin for QtumCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/slp.rs b/mm2src/coins/utxo/slp.rs index edb3809036..e200923887 100644 --- a/mm2src/coins/utxo/slp.rs +++ b/mm2src/coins/utxo/slp.rs @@ -1600,8 +1600,6 @@ impl From for TxFeeDetails { impl MmCoin for SlpToken { fn is_asset_chain(&self) -> bool { false } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.conf.abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/utxo_common_tests.rs b/mm2src/coins/utxo/utxo_common_tests.rs index bffe3d6a7b..d432207d8e 100644 --- a/mm2src/coins/utxo/utxo_common_tests.rs +++ b/mm2src/coins/utxo/utxo_common_tests.rs @@ -291,6 +291,7 @@ pub(super) async fn test_hd_utxo_tx_history_impl(rpc_client: ElectrumClient) { coin.clone(), storage, ctx.metrics.clone(), + ctx.event_stream_manager.clone(), current_balances.clone(), )); @@ -316,6 +317,7 @@ pub(super) async fn test_hd_utxo_tx_history_impl(rpc_client: ElectrumClient) { coin.clone(), storage, ctx.metrics.clone(), + ctx.event_stream_manager.clone(), current_balances, )); diff --git a/mm2src/coins/utxo/utxo_standard.rs b/mm2src/coins/utxo/utxo_standard.rs index eae749f10c..0abf28e35b 100644 --- a/mm2src/coins/utxo/utxo_standard.rs +++ b/mm2src/coins/utxo/utxo_standard.rs @@ -975,8 +975,6 @@ impl MarketCoinOps for UtxoStandardCoin { impl MmCoin for UtxoStandardCoin { fn is_asset_chain(&self) -> bool { utxo_common::is_asset_chain(&self.utxo_arc) } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn get_raw_transaction(&self, req: RawTransactionRequest) -> RawTransactionFut { diff --git a/mm2src/coins/utxo/utxo_tx_history_v2.rs b/mm2src/coins/utxo/utxo_tx_history_v2.rs index c60588e226..3f44593643 100644 --- a/mm2src/coins/utxo/utxo_tx_history_v2.rs +++ b/mm2src/coins/utxo/utxo_tx_history_v2.rs @@ -15,6 +15,7 @@ use common::log::{error, info}; use derive_more::Display; use keys::Address; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use mm2_state_machine::prelude::*; @@ -146,6 +147,8 @@ struct UtxoTxHistoryStateMachine`] everywhere. balances: HashMap, @@ -590,7 +593,6 @@ where let my_addresses = try_or_stop_unknown!(ctx.coin.my_addresses().await, "Error on getting my addresses"); - let streaming_manager = ctx.coin.get_ctx().unwrap().event_stream_manager.clone(); for (tx_hash, height) in self.all_tx_ids_with_height { let tx_hash_string = format!("{:02x}", tx_hash); match ctx.storage.history_has_tx_hash(&wallet_id, &tx_hash_string).await { @@ -622,7 +624,7 @@ where }, }; - streaming_manager + ctx.streaming_manager .send_fn(&TxHistoryEventStreamer::derive_streamer_id(ctx.coin.ticker()), || { tx_details.clone() }) @@ -715,6 +717,7 @@ pub async fn bch_and_slp_history_loop( coin: BchCoin, storage: impl TxHistoryStorage, metrics: MetricsArc, + streaming_manager: StreamingManager, current_balance: Option, ) { let balances = match current_balance { @@ -751,6 +754,7 @@ pub async fn bch_and_slp_history_loop( coin, storage, metrics, + streaming_manager, balances, }; state_machine @@ -763,6 +767,7 @@ pub async fn utxo_history_loop( coin: Coin, storage: Storage, metrics: MetricsArc, + streaming_manager: StreamingManager, current_balances: HashMap, ) where Coin: UtxoTxHistoryOps, @@ -772,6 +777,7 @@ pub async fn utxo_history_loop( coin, storage, metrics, + streaming_manager, balances: current_balances, }; state_machine diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 805d18d118..1e4ef60567 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1662,8 +1662,6 @@ impl WatcherOps for ZCoin { impl MmCoin for ZCoin { fn is_asset_chain(&self) -> bool { self.utxo_arc.conf.asset_chain } - fn get_ctx(&self) -> Option { MmArc::from_weak(&self.as_ref().ctx) } - fn spawner(&self) -> WeakSpawner { self.as_ref().abortable_system.weak_spawner() } fn withdraw(&self, _req: WithdrawRequest) -> WithdrawFut { diff --git a/mm2src/coins_activation/src/bch_with_tokens_activation.rs b/mm2src/coins_activation/src/bch_with_tokens_activation.rs index df05578354..b38c1bee36 100644 --- a/mm2src/coins_activation/src/bch_with_tokens_activation.rs +++ b/mm2src/coins_activation/src/bch_with_tokens_activation.rs @@ -346,7 +346,13 @@ impl PlatformCoinWithTokensActivationOps for BchCoin { storage: impl TxHistoryStorage + Send + 'static, initial_balance: Option, ) { - let fut = bch_and_slp_history_loop(self.clone(), storage, ctx.metrics.clone(), initial_balance); + let fut = bch_and_slp_history_loop( + self.clone(), + storage, + ctx.metrics.clone(), + ctx.event_stream_manager.clone(), + initial_balance, + ); let settings = AbortSettings::info_on_abort(format!("bch_and_slp_history_loop stopped for {}", self.ticker())); self.spawner().spawn_with_settings(fut, settings); diff --git a/mm2src/coins_activation/src/sia_coin_activation.rs b/mm2src/coins_activation/src/sia_coin_activation.rs index 11c72955ab..110f8bbb7b 100644 --- a/mm2src/coins_activation/src/sia_coin_activation.rs +++ b/mm2src/coins_activation/src/sia_coin_activation.rs @@ -17,6 +17,7 @@ use derive_more::Display; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use rpc_task::RpcTaskError; @@ -25,6 +26,7 @@ use serde_derive::Serialize; use serde_json::Value as Json; use std::collections::HashMap; use std::time::Duration; + pub type SiaCoinTaskManagerShared = InitStandaloneCoinTaskManagerShared; pub type SiaCoinRpcTaskHandleShared = InitStandaloneCoinTaskHandleShared; pub type SiaCoinAwaitingStatus = HwRpcTaskAwaitingStatus; @@ -237,6 +239,7 @@ impl InitStandaloneCoinActivationOps for SiaCoin { &self, _metrics: MetricsArc, _storage: impl TxHistoryStorage, + _streaming_manager: StreamingManager, _current_balances: HashMap, ) { } diff --git a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs index c4e7fa8c92..f1dbea7b81 100644 --- a/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs +++ b/mm2src/coins_activation/src/standalone_coin/init_standalone_coin.rs @@ -10,6 +10,7 @@ use coins::{lp_coinfind, lp_register_coin, CoinsContext, MmCoinEnum, RegisterCoi use common::{log, SuccessResponse}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use rpc_task::rpc_common::{CancelRpcTaskRequest, InitRpcTaskResponse, RpcTaskStatusRequest, RpcTaskUserActionRequest}; @@ -73,6 +74,7 @@ pub trait InitStandaloneCoinActivationOps: Into + Send + Sync + 'sta &self, metrics: MetricsArc, storage: impl TxHistoryStorage, + streaming_manager: StreamingManager, current_balances: HashMap, ); } @@ -217,6 +219,7 @@ where coin.start_history_background_fetching( self.ctx.metrics.clone(), TxHistoryStorageBuilder::new(&self.ctx).build()?, + self.ctx.event_stream_manager.clone(), current_balances, ); } diff --git a/mm2src/coins_activation/src/utxo_activation/common_impl.rs b/mm2src/coins_activation/src/utxo_activation/common_impl.rs index d5c9be6694..edba083c49 100644 --- a/mm2src/coins_activation/src/utxo_activation/common_impl.rs +++ b/mm2src/coins_activation/src/utxo_activation/common_impl.rs @@ -15,6 +15,7 @@ use crypto::{CryptoCtxError, HwRpcError}; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use std::collections::HashMap; @@ -99,6 +100,7 @@ pub(crate) fn start_history_background_fetching( coin: Coin, metrics: MetricsArc, storage: impl TxHistoryStorage, + streaming_manager: StreamingManager, current_balances: HashMap, ) where Coin: AsRef + UtxoTxHistoryOps, @@ -106,7 +108,7 @@ pub(crate) fn start_history_background_fetching( let spawner = coin.as_ref().abortable_system.weak_spawner(); let msg = format!("'utxo_history_loop' has been aborted for {}", coin.ticker()); - let fut = utxo_history_loop(coin, storage, metrics, current_balances); + let fut = utxo_history_loop(coin, storage, metrics, streaming_manager, current_balances); let settings = AbortSettings::info_on_abort(msg); spawner.spawn_with_settings(fut, settings); diff --git a/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs index 07574944ba..8c27226959 100644 --- a/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs +++ b/mm2src/coins_activation/src/utxo_activation/init_bch_activation.rs @@ -16,6 +16,7 @@ use coins::utxo::utxo_builder::{UtxoArcBuilder, UtxoCoinBuilder}; use coins::CoinProtocol; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde_json::Value as Json; @@ -109,8 +110,9 @@ impl InitStandaloneCoinActivationOps for BchCoin { &self, metrics: MetricsArc, storage: impl TxHistoryStorage, + streaming_manager: StreamingManager, current_balances: HashMap, ) { - start_history_background_fetching(self.clone(), metrics, storage, current_balances) + start_history_background_fetching(self.clone(), metrics, storage, streaming_manager, current_balances) } } diff --git a/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs index ae8cdec6ce..a6644f3275 100644 --- a/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs +++ b/mm2src/coins_activation/src/utxo_activation/init_qtum_activation.rs @@ -16,6 +16,7 @@ use coins::utxo::UtxoActivationParams; use coins::CoinProtocol; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde_json::Value as Json; @@ -83,8 +84,9 @@ impl InitStandaloneCoinActivationOps for QtumCoin { &self, metrics: MetricsArc, storage: impl TxHistoryStorage, + streaming_manager: StreamingManager, current_balances: HashMap, ) { - start_history_background_fetching(self.clone(), metrics, storage, current_balances) + start_history_background_fetching(self.clone(), metrics, storage, streaming_manager, current_balances) } } diff --git a/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs b/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs index 206c750f15..10715e2f0e 100644 --- a/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs +++ b/mm2src/coins_activation/src/utxo_activation/init_utxo_standard_activation.rs @@ -17,6 +17,7 @@ use coins::CoinProtocol; use futures::StreamExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use serde_json::Value as Json; @@ -124,8 +125,9 @@ impl InitStandaloneCoinActivationOps for UtxoStandardCoin { &self, metrics: MetricsArc, storage: impl TxHistoryStorage, + streaming_manager: StreamingManager, current_balances: HashMap, ) { - start_history_background_fetching(self.clone(), metrics, storage, current_balances) + start_history_background_fetching(self.clone(), metrics, storage, streaming_manager, current_balances) } } diff --git a/mm2src/coins_activation/src/z_coin_activation.rs b/mm2src/coins_activation/src/z_coin_activation.rs index 70da5c4eae..2332710218 100644 --- a/mm2src/coins_activation/src/z_coin_activation.rs +++ b/mm2src/coins_activation/src/z_coin_activation.rs @@ -16,6 +16,7 @@ use derive_more::Display; use futures::compat::Future01CompatExt; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; +use mm2_event_stream::StreamingManager; use mm2_metrics::MetricsArc; use mm2_number::BigDecimal; use rpc_task::RpcTaskError; @@ -303,6 +304,7 @@ impl InitStandaloneCoinActivationOps for ZCoin { &self, _metrics: MetricsArc, _storage: impl TxHistoryStorage, + _streaming_manager: StreamingManager, _current_balances: HashMap, ) { } From e65ace0b074567a5f003fc4cdf947a229475bbb2 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Wed, 6 Nov 2024 10:48:56 +0100 Subject: [PATCH 126/128] review(onur): do the same for tendermint history state machine --- mm2src/coins/tendermint/tendermint_tx_history_v2.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs index 6a06d000f4..a227cc96ad 100644 --- a/mm2src/coins/tendermint/tendermint_tx_history_v2.rs +++ b/mm2src/coins/tendermint/tendermint_tx_history_v2.rs @@ -128,6 +128,7 @@ impl CoinWithTxHistoryV2 for TendermintToken { struct TendermintTxHistoryStateMachine { coin: Coin, storage: Storage, + streaming_manager: StreamingManager, balances: AllBalancesResult, last_received_page: u32, last_spent_page: u32, @@ -573,6 +574,7 @@ where address: String, coin: &Coin, storage: &Storage, + streaming_manager: &StreamingManager, query: String, from_height: u64, page: &mut u32, @@ -589,7 +591,6 @@ where "could not get rpc client" ); - let streaming_manager: StreamingManager = panic!(); loop { let response = try_or_return_stopped_as_err!( client @@ -788,6 +789,7 @@ where self.address.clone(), &ctx.coin, &ctx.storage, + &ctx.streaming_manager, q, self.from_block_height, &mut ctx.last_spent_page, @@ -810,6 +812,7 @@ where self.address.clone(), &ctx.coin, &ctx.storage, + &ctx.streaming_manager, q, self.from_block_height, &mut ctx.last_received_page, @@ -923,7 +926,7 @@ fn get_value_from_event_attributes(events: &[EventAttribute], tag: &str, base64_ pub async fn tendermint_history_loop( coin: TendermintCoin, storage: impl TxHistoryStorage, - _ctx: MmArc, + ctx: MmArc, _current_balance: Option, ) { let balances = match coin.get_all_balances().await { @@ -937,6 +940,7 @@ pub async fn tendermint_history_loop( let mut state_machine = TendermintTxHistoryStateMachine { coin, storage, + streaming_manager: ctx.event_stream_manager.clone(), balances, last_received_page: 1, last_spent_page: 1, From 5f775d6150647dd5bacf4f58b49012263a0b7fab Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 18 Nov 2024 10:59:10 +0100 Subject: [PATCH 127/128] review(sami): code suggestions --- mm2src/mm2_event_stream/src/manager.rs | 8 +++----- mm2src/mm2_event_stream/src/streamer.rs | 15 ++++++++------- mm2src/mm2_net/src/event_streaming/sse_handler.rs | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index b1d8e016f5..1fbd885603 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::streamer::spawn; use crate::{Event, EventStreamer}; use common::executor::abortable_queue::WeakSpawner; -use common::log; +use common::log::{error, LogOnError}; use common::on_drop_callback::OnDropCallback; use futures::channel::mpsc::UnboundedSender; @@ -90,7 +90,7 @@ impl ClientInfo { // Only `try_send` here. If the channel is full (client is slow), the message // will be dropped and the client won't receive it. // This avoids blocking the broadcast to other receivers. - self.channel.try_send(event).ok(); + self.channel.try_send(event).error_log(); } } @@ -297,9 +297,7 @@ impl StreamingManager { if let Some(streamer_info) = this.streamers.get_mut(&streamer_id) { streamer_info.remove_client(&client_id); } else { - log::error!( - "Client {client_id} was listening to a non-existent streamer {streamer_id}. This is a bug!" - ); + error!("Client {client_id} was listening to a non-existent streamer {streamer_id}. This is a bug!"); } // If there are no more listening clients, terminate the streamer. if this.streamers.get(&streamer_id).map(|info| info.clients.len()) == Some(0) { diff --git a/mm2src/mm2_event_stream/src/streamer.rs b/mm2src/mm2_event_stream/src/streamer.rs index cf63ab766f..6c319cb89c 100644 --- a/mm2src/mm2_event_stream/src/streamer.rs +++ b/mm2src/mm2_event_stream/src/streamer.rs @@ -66,12 +66,13 @@ where move |any_input_data| { let streamer_id = streamer_id.clone(); future::ready( - if let Ok(input_data) = any_input_data.downcast() { - Some(*input_data) - } else { - error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::<::DataInType>()); - None - } + any_input_data + .downcast() + .map(|input_data| *input_data) + .map_err(|_| { + error!("Couldn't downcast a received message to {}. This message wasn't intended to be sent to this streamer ({streamer_id}).", any::type_name::()); + }) + .ok(), ) } }); @@ -97,7 +98,7 @@ where })?; // If the handler takes no input data, return `None` for the data sender. - if any::TypeId::of::<::DataInType>() == any::TypeId::of::() { + if any::TypeId::of::() == any::TypeId::of::() { Ok((tx_shutdown, None)) } else { Ok((tx_shutdown, Some(any_data_sender))) diff --git a/mm2src/mm2_net/src/event_streaming/sse_handler.rs b/mm2src/mm2_net/src/event_streaming/sse_handler.rs index 534e32e614..cd5ba585b3 100644 --- a/mm2src/mm2_net/src/event_streaming/sse_handler.rs +++ b/mm2src/mm2_net/src/event_streaming/sse_handler.rs @@ -29,7 +29,7 @@ pub async fn handle_sse(request: Request, ctx_h: u32) -> Response { let event_stream_manager = ctx.event_stream_manager.clone(); let Ok(mut rx) = event_stream_manager.new_client(client_id) else { - return handle_internal_error(format!("ID already in use")).await + return handle_internal_error("ID already in use".to_string()).await }; let body = Body::wrap_stream(async_stream::stream! { while let Some(event) = rx.recv().await { From f5dbfddd2abc48293e558fff0936843b9e87b570 Mon Sep 17 00:00:00 2001 From: Omer Yacine Date: Mon, 18 Nov 2024 11:11:30 +0100 Subject: [PATCH 128/128] review(sami): let event getter return a reference to avoid copying --- mm2src/mm2_event_stream/src/event.rs | 4 ++-- mm2src/mm2_event_stream/src/manager.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/mm2_event_stream/src/event.rs b/mm2src/mm2_event_stream/src/event.rs index ae66456eea..306bbc9e49 100644 --- a/mm2src/mm2_event_stream/src/event.rs +++ b/mm2src/mm2_event_stream/src/event.rs @@ -40,8 +40,8 @@ impl Event { pub fn origin(&self) -> &str { &self.event_type } /// Returns the event type and message as a pair. - pub fn get(&self) -> (String, Json) { + pub fn get(&self) -> (String, &Json) { let prefix = if self.error { "ERROR:" } else { "" }; - (format!("{prefix}{}", self.event_type), self.message.clone()) + (format!("{prefix}{}", self.event_type), &self.message) } } diff --git a/mm2src/mm2_event_stream/src/manager.rs b/mm2src/mm2_event_stream/src/manager.rs index 1fbd885603..f9c895a2d1 100644 --- a/mm2src/mm2_event_stream/src/manager.rs +++ b/mm2src/mm2_event_stream/src/manager.rs @@ -464,7 +464,7 @@ mod tests { let event = client1.try_recv().unwrap(); assert_eq!(event.origin(), streamer_id); // It's an echo streamer, so the message should be the same. - assert_eq!(event.get().1, json!(msg)); + assert_eq!(event.get().1, &json!(msg)); } // If we send the wrong datatype (void here instead of String), the streamer should ignore it.