From 416ef8df4d4fdd82b010c9775f6c23036e042718 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 10 Dec 2024 13:42:27 +0100 Subject: [PATCH 01/42] create a retry manager struct --- Cargo.lock | 15 +++++++ Cargo.toml | 1 + src/events/mod.rs | 3 +- src/events/processor.rs | 19 ++++++-- src/events/retry.rs | 98 +++++++++++++++++++++++++++++++++++++++++ src/watcher.rs | 19 +++++++- tests/watcher/utils.rs | 8 +++- 7 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 src/events/retry.rs diff --git a/Cargo.lock b/Cargo.lock index deb63054..a5e0da9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -806,6 +806,20 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "deadpool" version = "0.9.5" @@ -2507,6 +2521,7 @@ dependencies = [ "chrono", "const_format", "criterion", + "dashmap", "dotenv", "env_logger", "httpc-test", diff --git a/Cargo.toml b/Cargo.toml index 4443b7cd..ba06d61d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ reqwest = "0.12.9" base32 = "0.5.1" blake3 = "1.5.5" url = "2.5.4" +dashmap = "6.1.0" [dev-dependencies] anyhow = "1.0.94" diff --git a/src/events/mod.rs b/src/events/mod.rs index c00d8555..3dfe6ba4 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -6,6 +6,7 @@ use uri::ParsedUri; pub mod handlers; pub mod processor; pub mod uri; +pub mod retry; #[derive(Debug, Clone)] enum ResourceType { @@ -40,7 +41,7 @@ enum ResourceType { // Look for the end pattern after the start index, or use the end of the string if not found #[derive(Debug, Clone)] -enum EventType { +pub enum EventType { Put, Del, } diff --git a/src/events/processor.rs b/src/events/processor.rs index 4e40ba50..b8aa545c 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,6 +1,9 @@ +use std::sync::Arc; use std::time::Duration; +use super::retry::SenderMessage; use super::Event; +use crate::events::retry::RetryEvent; use crate::types::DynError; use crate::types::PubkyId; use crate::{models::homeserver::Homeserver, Config}; @@ -8,17 +11,21 @@ use log::{debug, error, info}; use pkarr::mainline::dht::Testnet; use pubky::PubkyClient; use reqwest::Client; +use tokio::sync::mpsc::Sender; +use tokio::sync::Mutex; pub struct EventProcessor { pubky_client: PubkyClient, http_client: Client, - homeserver: Homeserver, + pub homeserver: Homeserver, limit: u32, max_retries: u64, + pub sender: Arc>> } impl EventProcessor { - pub async fn from_config(config: &Config) -> Result { + pub async fn from_config(config: &Config, tx: Arc>>) -> Result { + let pubky_client = Self::init_pubky_client(config); let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; @@ -35,6 +42,7 @@ impl EventProcessor { homeserver, limit, max_retries, + sender: tx }) } @@ -50,7 +58,7 @@ impl EventProcessor { } } - pub async fn test(testnet: &Testnet, homeserver_url: String) -> Self { + pub async fn test(testnet: &Testnet, homeserver_url: String, tx: Arc>>) -> Self { let id = PubkyId("test".to_string()); let homeserver = Homeserver::new(id, homeserver_url).await.unwrap(); Self { @@ -59,6 +67,7 @@ impl EventProcessor { homeserver, limit: 1000, max_retries: 3, + sender: tx } } @@ -112,6 +121,10 @@ impl EventProcessor { }; if let Some(event) = event { debug!("Processing event: {:?}", event); + let retry_event = RetryEvent::new(&event.uri, &event.event_type, None, None); + let sender = self.sender.lock().await; + // Send the message to the receiver + let _ = sender.send(SenderMessage::Add(self.homeserver.id.to_string(), retry_event)).await; self.handle_event_with_retry(event).await?; } } diff --git a/src/events/retry.rs b/src/events/retry.rs new file mode 100644 index 00000000..35ee23f3 --- /dev/null +++ b/src/events/retry.rs @@ -0,0 +1,98 @@ +use std::{collections::LinkedList, sync::Arc}; +use dashmap::DashMap; +use tokio::sync::{mpsc::{Receiver, Sender}, Mutex}; + +use super::EventType; + +pub const CHANNEL_BUFFER: usize = 1024; + +#[derive(Debug, Clone)] +pub struct RetryEvent { + pub uri: String, // URI of the resource + pub event_type: EventType, // Type of event (e.g., PUT, DEL) + pub timestamp: u64, // Unix timestamp when the event was received + pub dependency: Option, // Optional parent URI for dependency tracking + pub retry_count: u32, // Number of retries attempted + pub error_message: Option, // Optional field to track failure reasons +} + +impl RetryEvent { + pub fn new(uri: &String, event_type: &EventType, dependency: Option, error_message: Option) -> Self { + Self { + uri: uri.to_string(), + event_type: event_type.clone(), + // TODO: Add now unixtime + timestamp: 1733831902, + dependency, + retry_count: 0, + error_message, + } + } +} + +#[derive(Debug, Clone)] +pub enum SenderMessage { + Retry(String), // Retry events associated with this key + Add(String, RetryEvent), // Add a new RetryEvent to the fail_events +} + +pub struct RetryManager { + pub sender: Arc>>, + receiver: Arc>>, + fail_events: DashMap> +} + +/// Initializes a new `RetryManager` with a message-passing channel. +/// +/// This function sets up the `RetryManager` by taking a tuple containing +/// a `Sender` and `Receiver` from a multi-producer, single-consumer (mpsc) channel. +/// +/// # Parameters +/// - `(tx, rx)`: A tuple containing: +/// - `tx` (`Sender`): The sending half of the mpsc channel, used to dispatch messages to the manager. +/// - `rx` (`Receiver`): The receiving half of the mpsc channel, used to listen for incoming messages. +impl RetryManager { + pub fn initialise((tx, rx): (Sender, Receiver)) -> Self { + Self { + receiver: Arc::new(Mutex::new(rx)), + sender: Arc::new(Mutex::new(tx)), + fail_events: DashMap::new() + } + } + + pub async fn exec(&self) { + let mut rx = self.receiver.lock().await; + while let Some(message) = rx.recv().await { + match message { + SenderMessage::Retry(homeserver_pubky) => { + self.retry_events_for_homeserver(&homeserver_pubky).await; + } + SenderMessage::Add(homeserver_pubky, retry_event) => { + self.add_fail_event(homeserver_pubky, retry_event); + } + } + } + } + + async fn retry_events_for_homeserver(&self, homeserver_pubky: &str) { + if let Some(retry_events) = self.fail_events.get(homeserver_pubky) { + println!("** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", homeserver_pubky); + for event in retry_events.iter() { + println!("Event URI: {}", event.uri); + } + } else { + println!("No retry events found for key: {}", homeserver_pubky); + } + } + + + + fn add_fail_event(&self, homeserver_pubky: String, retry_event: RetryEvent) { + let mut list = self + .fail_events + .entry(homeserver_pubky.clone()) + .or_insert_with(LinkedList::new); + list.push_back(retry_event); + println!("** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", homeserver_pubky); + } +} \ No newline at end of file diff --git a/src/watcher.rs b/src/watcher.rs index 9d1526cc..0f2a088c 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -1,6 +1,10 @@ use log::error; use log::info; +use pubky_nexus::events::retry::RetryManager; +use pubky_nexus::events::retry::SenderMessage; +use pubky_nexus::events::retry::CHANNEL_BUFFER; use pubky_nexus::{setup, Config, EventProcessor}; +use tokio::sync::mpsc; use tokio::time::{sleep, Duration}; /// Watches over a homeserver `/events` and writes into the Nexus databases @@ -8,7 +12,18 @@ use tokio::time::{sleep, Duration}; async fn main() -> Result<(), Box> { let config = Config::from_env(); setup(&config).await; - let mut event_processor = EventProcessor::from_config(&config).await?; + + let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = retry_manager.sender.clone(); + + // Create new asynchronous task to control the failed events + tokio::spawn(async move { + retry_manager.exec().await; + }); + + let mut event_processor = + EventProcessor::from_config(&config, sender_clone).await?; loop { info!("Fetching events..."); @@ -17,5 +32,7 @@ async fn main() -> Result<(), Box> { } // Wait for X milliseconds before fetching events again sleep(Duration::from_millis(config.watcher_sleep)).await; + let sender = event_processor.sender.lock().await; + let _ = sender.send(SenderMessage::Retry(event_processor.homeserver.id.to_string())).await; } } diff --git a/tests/watcher/utils.rs b/tests/watcher/utils.rs index 6282d951..12e614de 100644 --- a/tests/watcher/utils.rs +++ b/tests/watcher/utils.rs @@ -7,8 +7,9 @@ use pubky_app_specs::{ }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; -use pubky_nexus::{setup, Config, EventProcessor}; +use pubky_nexus::{events::retry::{RetryManager, CHANNEL_BUFFER}, setup, Config, EventProcessor}; use serde_json::to_vec; +use tokio::sync::mpsc; /// Struct to hold the setup environment for tests pub struct WatcherTest { @@ -28,7 +29,10 @@ impl WatcherTest { let homeserver = Homeserver::start_test(&testnet).await?; let client = PubkyClient::test(&testnet); let homeserver_url = format!("http://localhost:{}", homeserver.port()); - let event_processor = EventProcessor::test(&testnet, homeserver_url).await; + + let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); + + let event_processor = EventProcessor::test(&testnet, homeserver_url, retry_manager.sender.clone()).await; Ok(Self { config, From 950613424bcb2167ec9d3d1be86a3125683a3ec5 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 11 Dec 2024 12:57:46 +0100 Subject: [PATCH 02/42] prepare the WatcherTest for RetryManager component --- src/db/graph/queries/get.rs | 2 -- src/events/mod.rs | 4 ++-- src/events/processor.rs | 20 ++++++++++++-------- src/events/retry.rs | 16 ++++++++++------ tests/service/stream/post/reach/utils.rs | 2 -- tests/watcher/utils.rs | 9 ++++++++- 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/db/graph/queries/get.rs b/src/db/graph/queries/get.rs index c7096fa0..31acb98c 100644 --- a/src/db/graph/queries/get.rs +++ b/src/db/graph/queries/get.rs @@ -591,8 +591,6 @@ pub fn post_stream( cypher.push_str(&format!("LIMIT {}\n", limit)); } - println!("{:?}", cypher); - // Build the query and apply parameters using `param` method build_query_with_params(&cypher, &source, tags, kind, &pagination) } diff --git a/src/events/mod.rs b/src/events/mod.rs index 3dfe6ba4..95e91461 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -55,7 +55,7 @@ pub struct Event { } impl Event { - fn from_str( + pub fn from_str( line: &str, pubky_client: PubkyClient, ) -> Result, Box> { @@ -123,7 +123,7 @@ impl Event { })) } - async fn handle(self) -> Result<(), Box> { + pub async fn handle(self) -> Result<(), Box> { match self.event_type { EventType::Put => self.handle_put_event().await, EventType::Del => self.handle_del_event().await, diff --git a/src/events/processor.rs b/src/events/processor.rs index b8aa545c..efb1cf08 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,6 +1,6 @@ -use std::sync::Arc; use std::time::Duration; +use super::retry::SenderChannel; use super::retry::SenderMessage; use super::Event; use crate::events::retry::RetryEvent; @@ -11,8 +11,6 @@ use log::{debug, error, info}; use pkarr::mainline::dht::Testnet; use pubky::PubkyClient; use reqwest::Client; -use tokio::sync::mpsc::Sender; -use tokio::sync::Mutex; pub struct EventProcessor { pubky_client: PubkyClient, @@ -20,11 +18,11 @@ pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, max_retries: u64, - pub sender: Arc>> + pub sender: SenderChannel } impl EventProcessor { - pub async fn from_config(config: &Config, tx: Arc>>) -> Result { + pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { let pubky_client = Self::init_pubky_client(config); let homeserver = Homeserver::from_config(config).await?; @@ -58,7 +56,7 @@ impl EventProcessor { } } - pub async fn test(testnet: &Testnet, homeserver_url: String, tx: Arc>>) -> Self { + pub async fn test(testnet: &Testnet, homeserver_url: String, tx: SenderChannel) -> Self { let id = PubkyId("test".to_string()); let homeserver = Homeserver::new(id, homeserver_url).await.unwrap(); Self { @@ -121,10 +119,16 @@ impl EventProcessor { }; if let Some(event) = event { debug!("Processing event: {:?}", event); + // Experimental Block -> let retry_event = RetryEvent::new(&event.uri, &event.event_type, None, None); let sender = self.sender.lock().await; - // Send the message to the receiver - let _ = sender.send(SenderMessage::Add(self.homeserver.id.to_string(), retry_event)).await; + // Send the message to the receiver + let d = sender.send(SenderMessage::Add(self.homeserver.id.to_string(), retry_event)).await; + match d { + Ok(_) => (), + Err(e) => println!("ERROR{:?}", e.to_string()) + } + // <- Closing experimental block for sender channel self.handle_event_with_retry(event).await?; } } diff --git a/src/events/retry.rs b/src/events/retry.rs index 35ee23f3..addd5aa4 100644 --- a/src/events/retry.rs +++ b/src/events/retry.rs @@ -1,5 +1,6 @@ use std::{collections::LinkedList, sync::Arc}; use dashmap::DashMap; +use log::{debug, info}; use tokio::sync::{mpsc::{Receiver, Sender}, Mutex}; use super::EventType; @@ -36,9 +37,12 @@ pub enum SenderMessage { Add(String, RetryEvent), // Add a new RetryEvent to the fail_events } +pub type SenderChannel = Arc>>; +type ReceiverChannel = Arc>>; + pub struct RetryManager { - pub sender: Arc>>, - receiver: Arc>>, + pub sender: SenderChannel, + receiver: ReceiverChannel, fail_events: DashMap> } @@ -76,12 +80,12 @@ impl RetryManager { async fn retry_events_for_homeserver(&self, homeserver_pubky: &str) { if let Some(retry_events) = self.fail_events.get(homeserver_pubky) { - println!("** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", homeserver_pubky); + info!("** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", homeserver_pubky); for event in retry_events.iter() { - println!("Event URI: {}", event.uri); + info!("-> {:?}:{}",event.event_type, event.uri); } } else { - println!("No retry events found for key: {}", homeserver_pubky); + info!("No retry events found for key: {}", homeserver_pubky); } } @@ -93,6 +97,6 @@ impl RetryManager { .entry(homeserver_pubky.clone()) .or_insert_with(LinkedList::new); list.push_back(retry_event); - println!("** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", homeserver_pubky); + debug!("** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", homeserver_pubky); } } \ No newline at end of file diff --git a/tests/service/stream/post/reach/utils.rs b/tests/service/stream/post/reach/utils.rs index c9b54fe2..31ba22ed 100644 --- a/tests/service/stream/post/reach/utils.rs +++ b/tests/service/stream/post/reach/utils.rs @@ -39,8 +39,6 @@ pub async fn test_reach_filter_with_posts( path.push_str(&format!("&limit={}", limit)); } - println!("PATH: {:?}", path); - let body = make_request(&path).await?; if verify_timeline { diff --git a/tests/watcher/utils.rs b/tests/watcher/utils.rs index 12e614de..91e9d333 100644 --- a/tests/watcher/utils.rs +++ b/tests/watcher/utils.rs @@ -32,7 +32,14 @@ impl WatcherTest { let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); - let event_processor = EventProcessor::test(&testnet, homeserver_url, retry_manager.sender.clone()).await; + + let sender_clone = retry_manager.sender.clone(); + + tokio::spawn(async move { + retry_manager.exec().await; + }); + + let event_processor = EventProcessor::test(&testnet, homeserver_url, sender_clone).await; Ok(Self { config, From 04b79fff2d6b7bd4adcf0b6c7ebd88ba2abf3afd Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 11 Dec 2024 20:24:51 +0100 Subject: [PATCH 03/42] start catching the failed events --- src/events/processor.rs | 30 +++++++++++------ src/events/retry.rs | 74 ++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/events/processor.rs b/src/events/processor.rs index efb1cf08..9e0db612 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -21,6 +21,12 @@ pub struct EventProcessor { pub sender: SenderChannel } +#[derive(Debug, Clone)] +pub enum EventErrorType { + NotResolveHomeserver, + PubkyClientError +} + impl EventProcessor { pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { @@ -119,16 +125,6 @@ impl EventProcessor { }; if let Some(event) = event { debug!("Processing event: {:?}", event); - // Experimental Block -> - let retry_event = RetryEvent::new(&event.uri, &event.event_type, None, None); - let sender = self.sender.lock().await; - // Send the message to the receiver - let d = sender.send(SenderMessage::Add(self.homeserver.id.to_string(), retry_event)).await; - match d { - Ok(_) => (), - Err(e) => println!("ERROR{:?}", e.to_string()) - } - // <- Closing experimental block for sender channel self.handle_event_with_retry(event).await?; } } @@ -150,6 +146,20 @@ impl EventProcessor { "Error while handling event after {} attempts: {}", attempts, e ); + + // Send the failed event to the retry manager to retry indexing + let mut error_type = None; + + if e.to_string() == "Generic error: Could not resolve homeserver" { + error_type = Some(EventErrorType::NotResolveHomeserver); + } else if e.to_string().contains("error sending request for url") { + error_type = Some(EventErrorType::PubkyClientError); + } + if let Some(error) = error_type { + let fail_event = RetryEvent::new(&event.uri, &event.event_type, None, error); + let sender = self.sender.lock().await; + sender.send(SenderMessage::Add(self.homeserver.id.clone(), fail_event)).await?; + } break Ok(()); } else { error!( diff --git a/src/events/retry.rs b/src/events/retry.rs index addd5aa4..e0d8b4f9 100644 --- a/src/events/retry.rs +++ b/src/events/retry.rs @@ -1,40 +1,50 @@ -use std::{collections::LinkedList, sync::Arc}; use dashmap::DashMap; use log::{debug, info}; -use tokio::sync::{mpsc::{Receiver, Sender}, Mutex}; +use std::{collections::LinkedList, sync::Arc}; +use tokio::sync::{ + mpsc::{Receiver, Sender}, + Mutex, +}; + +use crate::types::PubkyId; -use super::EventType; +use super::{processor::EventErrorType, EventType}; pub const CHANNEL_BUFFER: usize = 1024; #[derive(Debug, Clone)] pub struct RetryEvent { - pub uri: String, // URI of the resource - pub event_type: EventType, // Type of event (e.g., PUT, DEL) - pub timestamp: u64, // Unix timestamp when the event was received - pub dependency: Option, // Optional parent URI for dependency tracking - pub retry_count: u32, // Number of retries attempted - pub error_message: Option, // Optional field to track failure reasons + pub uri: String, // URI of the resource + pub event_type: EventType, // Type of event (e.g., PUT, DEL) + pub timestamp: u64, // Unix timestamp when the event was received + pub dependency: Option, // Optional parent URI for dependency tracking + pub retry_count: u32, // Number of retries attempted + pub error_type: EventErrorType, // Optional field to track failure reasons } impl RetryEvent { - pub fn new(uri: &String, event_type: &EventType, dependency: Option, error_message: Option) -> Self { + pub fn new( + uri: &String, + event_type: &EventType, + dependency: Option, + error_type: EventErrorType + ) -> Self { Self { - uri: uri.to_string(), - event_type: event_type.clone(), - // TODO: Add now unixtime - timestamp: 1733831902, - dependency, - retry_count: 0, - error_message, + uri: uri.to_string(), + event_type: event_type.clone(), + // TODO: Add now unixtime + timestamp: 1733831902, + dependency, + retry_count: 0, + error_type, } } } #[derive(Debug, Clone)] pub enum SenderMessage { - Retry(String), // Retry events associated with this key - Add(String, RetryEvent), // Add a new RetryEvent to the fail_events + Retry(String), // Retry events associated with this key + Add(PubkyId, RetryEvent), // Add a new RetryEvent to the fail_events } pub type SenderChannel = Arc>>; @@ -43,12 +53,12 @@ type ReceiverChannel = Arc>>; pub struct RetryManager { pub sender: SenderChannel, receiver: ReceiverChannel, - fail_events: DashMap> + fail_events: DashMap>, } /// Initializes a new `RetryManager` with a message-passing channel. /// -/// This function sets up the `RetryManager` by taking a tuple containing +/// This function sets up the `RetryManager` by taking a tuple containing /// a `Sender` and `Receiver` from a multi-producer, single-consumer (mpsc) channel. /// /// # Parameters @@ -60,7 +70,7 @@ impl RetryManager { Self { receiver: Arc::new(Mutex::new(rx)), sender: Arc::new(Mutex::new(tx)), - fail_events: DashMap::new() + fail_events: DashMap::new(), } } @@ -80,23 +90,27 @@ impl RetryManager { async fn retry_events_for_homeserver(&self, homeserver_pubky: &str) { if let Some(retry_events) = self.fail_events.get(homeserver_pubky) { - info!("** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", homeserver_pubky); + info!( + "** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", + homeserver_pubky + ); for event in retry_events.iter() { - info!("-> {:?}:{}",event.event_type, event.uri); + info!("-> {:?}:{}:{:?}", event.event_type, event.uri, event.error_type); } } else { info!("No retry events found for key: {}", homeserver_pubky); } } - - - fn add_fail_event(&self, homeserver_pubky: String, retry_event: RetryEvent) { + fn add_fail_event(&self, homeserver_pubky: PubkyId, retry_event: RetryEvent) { let mut list = self .fail_events - .entry(homeserver_pubky.clone()) + .entry(homeserver_pubky.to_string()) .or_insert_with(LinkedList::new); list.push_back(retry_event); - debug!("** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", homeserver_pubky); + debug!( + "** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", + homeserver_pubky + ); } -} \ No newline at end of file +} From cdac5fd50ed1ec40fedbe3ca5b2c78eb3932b8cd Mon Sep 17 00:00:00 2001 From: tipogi Date: Thu, 12 Dec 2024 12:07:38 +0100 Subject: [PATCH 04/42] minor changes --- src/events/mod.rs | 2 +- src/events/processor.rs | 20 +++++++++++--------- src/events/retry.rs | 38 +++++++++++++++++++++----------------- src/watcher.rs | 21 +++++++++++++++++---- tests/watcher/utils.rs | 6 ++++-- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/src/events/mod.rs b/src/events/mod.rs index 95e91461..18f36ece 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -5,8 +5,8 @@ use uri::ParsedUri; pub mod handlers; pub mod processor; -pub mod uri; pub mod retry; +pub mod uri; #[derive(Debug, Clone)] enum ResourceType { diff --git a/src/events/processor.rs b/src/events/processor.rs index 9e0db612..cc46f5f7 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -18,18 +18,17 @@ pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, max_retries: u64, - pub sender: SenderChannel + pub sender: SenderChannel, } #[derive(Debug, Clone)] pub enum EventErrorType { NotResolveHomeserver, - PubkyClientError + PubkyClientError, } impl EventProcessor { pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { - let pubky_client = Self::init_pubky_client(config); let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; @@ -46,7 +45,7 @@ impl EventProcessor { homeserver, limit, max_retries, - sender: tx + sender: tx, }) } @@ -71,7 +70,7 @@ impl EventProcessor { homeserver, limit: 1000, max_retries: 3, - sender: tx + sender: tx, } } @@ -153,12 +152,15 @@ impl EventProcessor { if e.to_string() == "Generic error: Could not resolve homeserver" { error_type = Some(EventErrorType::NotResolveHomeserver); } else if e.to_string().contains("error sending request for url") { - error_type = Some(EventErrorType::PubkyClientError); - } + error_type = Some(EventErrorType::PubkyClientError); + } if let Some(error) = error_type { - let fail_event = RetryEvent::new(&event.uri, &event.event_type, None, error); + let fail_event = + RetryEvent::new(&event.uri, &event.event_type, None, error); let sender = self.sender.lock().await; - sender.send(SenderMessage::Add(self.homeserver.id.clone(), fail_event)).await?; + sender + .send(SenderMessage::Add(self.homeserver.id.clone(), fail_event)) + .await?; } break Ok(()); } else { diff --git a/src/events/retry.rs b/src/events/retry.rs index e0d8b4f9..b27046ee 100644 --- a/src/events/retry.rs +++ b/src/events/retry.rs @@ -1,5 +1,5 @@ use dashmap::DashMap; -use log::{debug, info}; +use log::debug; use std::{collections::LinkedList, sync::Arc}; use tokio::sync::{ mpsc::{Receiver, Sender}, @@ -14,11 +14,11 @@ pub const CHANNEL_BUFFER: usize = 1024; #[derive(Debug, Clone)] pub struct RetryEvent { - pub uri: String, // URI of the resource - pub event_type: EventType, // Type of event (e.g., PUT, DEL) - pub timestamp: u64, // Unix timestamp when the event was received - pub dependency: Option, // Optional parent URI for dependency tracking - pub retry_count: u32, // Number of retries attempted + pub uri: String, // URI of the resource + pub event_type: EventType, // Type of event (e.g., PUT, DEL) + pub timestamp: u64, // Unix timestamp when the event was received + pub dependency: Option, // Optional parent URI for dependency tracking + pub retry_count: u32, // Number of retries attempted pub error_type: EventErrorType, // Optional field to track failure reasons } @@ -27,7 +27,7 @@ impl RetryEvent { uri: &String, event_type: &EventType, dependency: Option, - error_type: EventErrorType + error_type: EventErrorType, ) -> Self { Self { uri: uri.to_string(), @@ -43,7 +43,7 @@ impl RetryEvent { #[derive(Debug, Clone)] pub enum SenderMessage { - Retry(String), // Retry events associated with this key + Retry(String), // Retry events associated with this key Add(PubkyId, RetryEvent), // Add a new RetryEvent to the fail_events } @@ -90,27 +90,31 @@ impl RetryManager { async fn retry_events_for_homeserver(&self, homeserver_pubky: &str) { if let Some(retry_events) = self.fail_events.get(homeserver_pubky) { - info!( - "** RETRY_MANAGER ===> Trying to fetch the failing events from {:?}", + debug!( + "Trying to fetch again the failing events from {:?}...", homeserver_pubky ); for event in retry_events.iter() { - info!("-> {:?}:{}:{:?}", event.event_type, event.uri, event.error_type); + debug!( + "-> {:?}:{}:{:?}", + event.event_type, event.uri, event.error_type + ); } } else { - info!("No retry events found for key: {}", homeserver_pubky); + debug!("No retry events found for key: {}", homeserver_pubky); } } fn add_fail_event(&self, homeserver_pubky: PubkyId, retry_event: RetryEvent) { + debug!( + "Added fail event in the HashMap: {:?}: {}", + retry_event.event_type, retry_event.uri + ); + // Write the event in the HashMap let mut list = self .fail_events .entry(homeserver_pubky.to_string()) - .or_insert_with(LinkedList::new); + .or_default(); list.push_back(retry_event); - debug!( - "** RETRY_MANAGER: Added fail event for homeserver_pubky: {}", - homeserver_pubky - ); } } diff --git a/src/watcher.rs b/src/watcher.rs index 0f2a088c..f8e495b6 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -7,6 +7,8 @@ use pubky_nexus::{setup, Config, EventProcessor}; use tokio::sync::mpsc; use tokio::time::{sleep, Duration}; +const RETRY_THRESHOLD: u8 = 5; + /// Watches over a homeserver `/events` and writes into the Nexus databases #[tokio::main] async fn main() -> Result<(), Box> { @@ -22,8 +24,11 @@ async fn main() -> Result<(), Box> { retry_manager.exec().await; }); - let mut event_processor = - EventProcessor::from_config(&config, sender_clone).await?; + let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; + + // Experimental. We need to think how/where achieve that + // Maybe add in .env file... + let mut retry_failed_events = 0; loop { info!("Fetching events..."); @@ -32,7 +37,15 @@ async fn main() -> Result<(), Box> { } // Wait for X milliseconds before fetching events again sleep(Duration::from_millis(config.watcher_sleep)).await; - let sender = event_processor.sender.lock().await; - let _ = sender.send(SenderMessage::Retry(event_processor.homeserver.id.to_string())).await; + retry_failed_events += 1; + + if RETRY_THRESHOLD == retry_failed_events { + let sender = event_processor.sender.lock().await; + let _ = sender + .send(SenderMessage::Retry( + event_processor.homeserver.id.to_string(), + )) + .await; + } } } diff --git a/tests/watcher/utils.rs b/tests/watcher/utils.rs index 91e9d333..e9be29a7 100644 --- a/tests/watcher/utils.rs +++ b/tests/watcher/utils.rs @@ -7,7 +7,10 @@ use pubky_app_specs::{ }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; -use pubky_nexus::{events::retry::{RetryManager, CHANNEL_BUFFER}, setup, Config, EventProcessor}; +use pubky_nexus::{ + events::retry::{RetryManager, CHANNEL_BUFFER}, + setup, Config, EventProcessor, +}; use serde_json::to_vec; use tokio::sync::mpsc; @@ -32,7 +35,6 @@ impl WatcherTest { let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); - let sender_clone = retry_manager.sender.clone(); tokio::spawn(async move { From a9ba37571406329fee3e84436cd2d00193566dbf Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 18 Dec 2024 15:22:44 +0100 Subject: [PATCH 05/42] Add TODO comment --- src/events/processor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/events/processor.rs b/src/events/processor.rs index 76f6f9bc..53606c90 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -157,6 +157,7 @@ impl EventProcessor { } else if e.to_string().contains("error sending request for url") { error_type = Some(EventErrorType::PubkyClientError); } + // TODO: Another else if to catch the error of the graph if let Some(error) = error_type { let fail_event = RetryEvent::new(&event.uri, &event.event_type, None, error); From b7832c50e0ce5aed84ba8509b7788c3159731b93 Mon Sep 17 00:00:00 2001 From: SHAcollision <127778313+SHAcollision@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:38:35 +0100 Subject: [PATCH 06/42] deps: bump axum to 0.8.1 (#276) --- Cargo.lock | 113 ++++++++++++++++++++-------- Cargo.toml | 16 +++- src/db/kv/traits.rs | 2 +- src/lib.rs | 1 - src/models/file/details.rs | 2 +- src/models/follow/followers.rs | 2 +- src/models/follow/following.rs | 2 +- src/models/follow/traits.rs | 2 +- src/models/tag/post.rs | 2 +- src/models/tag/stream.rs | 2 +- src/models/tag/traits/collection.rs | 2 +- src/models/tag/traits/taggers.rs | 2 +- src/models/tag/user.rs | 2 +- src/models/traits.rs | 2 +- src/models/user/details.rs | 2 +- src/models/user/follows.rs | 2 +- src/models/user/muted.rs | 2 +- src/routes/macros.rs | 9 --- src/routes/v0/file/mod.rs | 6 +- src/routes/v0/notification/mod.rs | 4 +- src/routes/v0/post/mod.rs | 14 ++-- src/routes/v0/search/mod.rs | 6 +- src/routes/v0/tag/mod.rs | 6 +- src/routes/v0/user/mod.rs | 22 +++--- 24 files changed, 137 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89c4e605..7add41f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,9 +153,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "1b1244b10dcd56c92219da4e14caa97e312079e185f04ba3eea25061561dc0a0" dependencies = [ "proc-macro2", "quote", @@ -190,7 +190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "axum-macros", "bytes", "futures-util", @@ -200,7 +200,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -210,9 +210,43 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "tokio", - "tower 0.5.1", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +dependencies = [ + "axum-core 0.5.0", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -233,7 +267,27 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -245,8 +299,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum", - "axum-core", + "axum 0.7.9", + "axum-core 0.4.5", "bytes", "futures-util", "headers", @@ -1887,6 +1941,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.4" @@ -2444,8 +2504,7 @@ dependencies = [ [[package]] name = "pubky-app-specs" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb66e94b59284b847f8ab438f7e589e91c2074b1282e027ade7e9909b349ef8" +source = "git+https://github.com/pubky/pubky-app-specs#b885faa516058137615b58b959d69160a26d4bde" dependencies = [ "base32", "blake3", @@ -2501,7 +2560,8 @@ name = "pubky-nexus" version = "0.2.0" dependencies = [ "anyhow", - "axum", + "async-trait", + "axum 0.8.1", "base32", "blake3", "chrono", @@ -2554,7 +2614,7 @@ version = "0.1.0" source = "git+https://github.com/pubky/pubky-core.git?tag=pubky-v0.3.0#aede289c2606204fe944919b5024ee8cefe32c50" dependencies = [ "anyhow", - "axum", + "axum 0.7.9", "axum-extra", "base32", "bytes", @@ -2831,7 +2891,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -3328,12 +3388,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.1" @@ -3648,14 +3702,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -3669,7 +3723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fd0118512cf0b3768f7fcccf0bef1ae41d68f2b45edc1e77432b36c97c56c6d" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "cookie", "futures-util", "http", @@ -3891,8 +3945,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utoipa" version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e76d357bc95c7d0939c92c04c9269871a8470eea39cb1f0231eeadb0c47d0f" +source = "git+https://github.com/juhaku/utoipa?rev=d522f744259dc4fde5f45d187983fb68c8167029#d522f744259dc4fde5f45d187983fb68c8167029" dependencies = [ "indexmap", "serde", @@ -3903,8 +3956,7 @@ dependencies = [ [[package]] name = "utoipa-gen" version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "564b03f8044ad6806bdc0d635e88be24967e785eef096df6b2636d2cc1e05d4b" +source = "git+https://github.com/juhaku/utoipa?rev=d522f744259dc4fde5f45d187983fb68c8167029#d522f744259dc4fde5f45d187983fb68c8167029" dependencies = [ "proc-macro2", "quote", @@ -3914,10 +3966,9 @@ dependencies = [ [[package]] name = "utoipa-swagger-ui" version = "8.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4b5ac679cc6dfc5ea3f2823b0291c777750ffd5e13b21137e0f7ac0e8f9617" +source = "git+https://github.com/juhaku/utoipa?rev=d522f744259dc4fde5f45d187983fb68c8167029#d522f744259dc4fde5f45d187983fb68c8167029" dependencies = [ - "axum", + "axum 0.8.1", "base64 0.22.1", "mime_guess", "regex", diff --git a/Cargo.toml b/Cargo.toml index 008a96dd..3f85865d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,16 +14,20 @@ pkarr = { git = "https://github.com/Pubky/pkarr", branch = "v3", package = "pkar "async", ] } pubky = "0.3.0" -pubky-app-specs = { version = "0.2.1", features = ["openapi"] } +pubky-app-specs = { git = "https://github.com/pubky/pubky-app-specs", features = [ + "openapi", +] } tokio = { version = "1.42.0", features = ["full"] } -axum = "0.7.9" +axum = "0.8.1" redis = { version = "0.27.6", features = ["tokio-comp", "json"] } neo4rs = "0.8.0" serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.134" once_cell = "1.20.2" -utoipa = "5.3.0" -utoipa-swagger-ui = { version = "8.1.0", features = ["axum"] } +utoipa = { git = "https://github.com/juhaku/utoipa", rev = "d522f744259dc4fde5f45d187983fb68c8167029" } +utoipa-swagger-ui = { git = "https://github.com/juhaku/utoipa", rev = "d522f744259dc4fde5f45d187983fb68c8167029", features = [ + "axum", +] } tower-http = { version = "0.6.2", features = ["fs", "cors"] } dotenv = "0.15" log = "0.4.22" @@ -36,6 +40,7 @@ reqwest = "0.12.9" base32 = "0.5.1" blake3 = "1.5.5" url = "2.5.4" +async-trait = "0.1.84" [dev-dependencies] anyhow = "1.0.95" @@ -46,6 +51,9 @@ rand = "0.8.5" rand_distr = "0.4.3" tokio-shared-rt = "0.1" +[patch.crates-io] +utoipa-swagger-ui = { git = "https://github.com/juhaku/utoipa", rev = "d522f744259dc4fde5f45d187983fb68c8167029" } + [lib] name = "pubky_nexus" path = "src/lib.rs" diff --git a/src/db/kv/traits.rs b/src/db/kv/traits.rs index 054601f1..6eda4682 100644 --- a/src/db/kv/traits.rs +++ b/src/db/kv/traits.rs @@ -1,6 +1,6 @@ use super::index::*; use crate::types::DynError; -use axum::async_trait; +use async_trait::async_trait; use json::JsonAction; use serde::{de::DeserializeOwned, Serialize}; use sorted_sets::{ScoreAction, SortOrder, SORTED_PREFIX}; diff --git a/src/lib.rs b/src/lib.rs index b8cfe803..72c5470c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,5 +20,4 @@ pub use events::processor::EventProcessor; pub use reindex::reindex; pub use setup::setup; -#[macro_use] extern crate const_format; diff --git a/src/models/file/details.rs b/src/models/file/details.rs index 7263f16c..736d7198 100644 --- a/src/models/file/details.rs +++ b/src/models/file/details.rs @@ -2,7 +2,7 @@ use crate::db::graph::exec::exec_single_row; use crate::models::traits::Collection; use crate::types::DynError; use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use chrono::Utc; use neo4rs::Query; use pubky_app_specs::PubkyAppFile; diff --git a/src/models/follow/followers.rs b/src/models/follow/followers.rs index a5f06677..41598463 100644 --- a/src/models/follow/followers.rs +++ b/src/models/follow/followers.rs @@ -1,5 +1,5 @@ use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use neo4rs::Query; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; diff --git a/src/models/follow/following.rs b/src/models/follow/following.rs index e44e3a14..44d50b0d 100644 --- a/src/models/follow/following.rs +++ b/src/models/follow/following.rs @@ -1,5 +1,5 @@ use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use neo4rs::Query; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; diff --git a/src/models/follow/traits.rs b/src/models/follow/traits.rs index 2dea81e2..a4647018 100644 --- a/src/models/follow/traits.rs +++ b/src/models/follow/traits.rs @@ -2,7 +2,7 @@ use crate::db::connectors::neo4j::get_neo4j_graph; use crate::db::graph::exec::exec_boolean_row; use crate::types::DynError; use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use chrono::Utc; use neo4rs::Query; diff --git a/src/models/tag/post.rs b/src/models/tag/post.rs index 819649b9..78c6df00 100644 --- a/src/models/tag/post.rs +++ b/src/models/tag/post.rs @@ -1,5 +1,5 @@ use crate::RedisOps; -use axum::async_trait; +use async_trait::async_trait; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; diff --git a/src/models/tag/stream.rs b/src/models/tag/stream.rs index 03be926e..02584b8e 100644 --- a/src/models/tag/stream.rs +++ b/src/models/tag/stream.rs @@ -1,7 +1,7 @@ use crate::db::graph::exec::retrieve_from_graph; use crate::db::kv::index::sorted_sets::SortOrder; use crate::types::{DynError, Timeframe}; -use axum::async_trait; +use async_trait::async_trait; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt::Display; diff --git a/src/models/tag/traits/collection.rs b/src/models/tag/traits/collection.rs index d6a560a1..9bb91992 100644 --- a/src/models/tag/traits/collection.rs +++ b/src/models/tag/traits/collection.rs @@ -1,5 +1,5 @@ use crate::types::DynError; -use axum::async_trait; +use async_trait::async_trait; use log::error; use neo4rs::Query; diff --git a/src/models/tag/traits/taggers.rs b/src/models/tag/traits/taggers.rs index af920419..f1fd954a 100644 --- a/src/models/tag/traits/taggers.rs +++ b/src/models/tag/traits/taggers.rs @@ -1,6 +1,6 @@ use crate::types::{DynError, Pagination}; use crate::RedisOps; -use axum::async_trait; +use async_trait::async_trait; use super::collection::CACHE_SET_PREFIX; diff --git a/src/models/tag/user.rs b/src/models/tag/user.rs index 2def5564..ffb7a2dd 100644 --- a/src/models/tag/user.rs +++ b/src/models/tag/user.rs @@ -1,5 +1,5 @@ use crate::RedisOps; -use axum::async_trait; +use async_trait::async_trait; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; diff --git a/src/models/traits.rs b/src/models/traits.rs index b9198115..08caf8df 100644 --- a/src/models/traits.rs +++ b/src/models/traits.rs @@ -1,4 +1,4 @@ -use axum::async_trait; +use async_trait::async_trait; use neo4rs::Query; use crate::db::connectors::neo4j::get_neo4j_graph; diff --git a/src/models/user/details.rs b/src/models/user/details.rs index 1b85864b..5a344bf8 100644 --- a/src/models/user/details.rs +++ b/src/models/user/details.rs @@ -4,7 +4,7 @@ use crate::models::traits::Collection; use crate::types::DynError; use crate::types::PubkyId; use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use chrono::Utc; use neo4rs::Query; use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; diff --git a/src/models/user/follows.rs b/src/models/user/follows.rs index 7d4c7042..196a7567 100644 --- a/src/models/user/follows.rs +++ b/src/models/user/follows.rs @@ -1,7 +1,7 @@ // use crate::db::connectors::neo4j::get_neo4j_graph; // use crate::db::graph::exec::exec_boolean_row; // use crate::{queries, RedisOps}; -// use axum::async_trait; +// use async_trait::async_trait; // use chrono::Utc; // use neo4rs::Query; // use serde::{Deserialize, Serialize}; diff --git a/src/models/user/muted.rs b/src/models/user/muted.rs index 1ff6df20..3e7dccb3 100644 --- a/src/models/user/muted.rs +++ b/src/models/user/muted.rs @@ -2,7 +2,7 @@ use crate::db::connectors::neo4j::get_neo4j_graph; use crate::db::graph::exec::exec_single_row; use crate::types::DynError; use crate::{queries, RedisOps}; -use axum::async_trait; +use async_trait::async_trait; use chrono::Utc; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; diff --git a/src/routes/macros.rs b/src/routes/macros.rs index f71d09e0..5540bf33 100644 --- a/src/routes/macros.rs +++ b/src/routes/macros.rs @@ -5,12 +5,3 @@ macro_rules! register_routes { $(.route($path, axum::routing::get($handler)))* }; } - -/// Transforms a "swagger-like" endpoint to "axum-like" manipulating &'static str -/// Example `"/v1/user/{user_id}"` -> `"/v1/user/:user_id"` -#[macro_export] -macro_rules! to_axum { - ($route:expr) => { - str_replace!(str_replace!($route, "{", ":"), "}", "") - }; -} diff --git a/src/routes/v0/file/mod.rs b/src/routes/v0/file/mod.rs index 369ecf88..e792a2a4 100644 --- a/src/routes/v0/file/mod.rs +++ b/src/routes/v0/file/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -8,10 +8,10 @@ mod list; pub fn routes() -> Router { let router = register_routes!(Router::new(), - to_axum!(endpoints::FILE_ROUTE) => details::file_details_handler, + endpoints::FILE_ROUTE => details::file_details_handler, ); router.route( - to_axum!(endpoints::FILE_LIST_ROUTE), + endpoints::FILE_LIST_ROUTE, axum::routing::post(list::file_details_by_uris_handler), ) } diff --git a/src/routes/v0/notification/mod.rs b/src/routes/v0/notification/mod.rs index d829c9e4..b95ca6fd 100644 --- a/src/routes/v0/notification/mod.rs +++ b/src/routes/v0/notification/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -7,7 +7,7 @@ mod list; pub fn routes() -> Router { register_routes!(Router::new(), - to_axum!(endpoints::NOTIFICATION_ROUTE) => list::list_notifications_handler + endpoints::NOTIFICATION_ROUTE => list::list_notifications_handler ) } diff --git a/src/routes/v0/post/mod.rs b/src/routes/v0/post/mod.rs index 55172c24..bb21a522 100644 --- a/src/routes/v0/post/mod.rs +++ b/src/routes/v0/post/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -11,12 +11,12 @@ mod view; pub fn routes() -> Router { register_routes!(Router::new(), - to_axum!(endpoints::POST_ROUTE) => view::post_view_handler, - to_axum!(endpoints::POST_DETAILS_ROUTE) => details::post_details_handler, - to_axum!(endpoints::POST_COUNTS_ROUTE) => counts::post_counts_handler, - to_axum!(endpoints::POST_BOOKMARK_ROUTE) => bookmark::post_bookmark_handler, - to_axum!(endpoints::POST_TAGS_ROUTE) => tags::post_tags_handler, - to_axum!(endpoints::POST_TAGGERS_ROUTE) => tags::post_taggers_handler, + endpoints::POST_ROUTE => view::post_view_handler, + endpoints::POST_DETAILS_ROUTE => details::post_details_handler, + endpoints::POST_COUNTS_ROUTE => counts::post_counts_handler, + endpoints::POST_BOOKMARK_ROUTE => bookmark::post_bookmark_handler, + endpoints::POST_TAGS_ROUTE => tags::post_tags_handler, + endpoints::POST_TAGGERS_ROUTE => tags::post_taggers_handler, ) } diff --git a/src/routes/v0/search/mod.rs b/src/routes/v0/search/mod.rs index 539932b4..fac7ce5e 100644 --- a/src/routes/v0/search/mod.rs +++ b/src/routes/v0/search/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -8,8 +8,8 @@ mod users; pub fn routes() -> Router { register_routes!(Router::new(), - to_axum!(endpoints::SEARCH_USERS_ROUTE) => users::search_users_handler, - to_axum!(endpoints::SEARCH_TAGS_ROUTE) => tags::search_post_tags_handler + endpoints::SEARCH_USERS_ROUTE => users::search_users_handler, + endpoints::SEARCH_TAGS_ROUTE => tags::search_post_tags_handler ) } diff --git a/src/routes/v0/tag/mod.rs b/src/routes/v0/tag/mod.rs index cb938cd7..28d56f45 100644 --- a/src/routes/v0/tag/mod.rs +++ b/src/routes/v0/tag/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -7,8 +7,8 @@ mod global; pub fn routes() -> Router { register_routes!(Router::new(), - to_axum!(endpoints::TAGS_HOT_ROUTE) => global::hot_tags_handler, - to_axum!(endpoints::TAG_TAGGERS_ROUTE) => global::tag_taggers_handler + endpoints::TAGS_HOT_ROUTE => global::hot_tags_handler, + endpoints::TAG_TAGGERS_ROUTE => global::tag_taggers_handler ) } diff --git a/src/routes/v0/user/mod.rs b/src/routes/v0/user/mod.rs index 8891c27e..93ba0e8e 100644 --- a/src/routes/v0/user/mod.rs +++ b/src/routes/v0/user/mod.rs @@ -1,5 +1,5 @@ +use crate::register_routes; use crate::routes::v0::endpoints; -use crate::{register_routes, to_axum}; use axum::Router; use utoipa::OpenApi; @@ -13,16 +13,16 @@ mod view; pub fn routes() -> Router { register_routes!(Router::new(), - to_axum!(endpoints::USER_ROUTE) => view::user_view_handler, - to_axum!(endpoints::USER_DETAILS_ROUTE) => details::user_details_handler, - to_axum!(endpoints::RELATIONSHIP_ROUTE) => relationship::user_relationship_handler, - to_axum!(endpoints::USER_TAGS_ROUTE) => tags::user_tags_handler, - to_axum!(endpoints::USER_TAGGERS_ROUTE) => tags::user_taggers_handler, - to_axum!(endpoints::USER_COUNTS_ROUTE) => counts::user_counts_handler, - to_axum!(endpoints::USER_FOLLOWERS_ROUTE) => follows::user_followers_handler, - to_axum!(endpoints::USER_FOLLOWING_ROUTE) => follows::user_following_handler, - to_axum!(endpoints::USER_FRIENDS_ROUTE) => follows::user_friends_handler, - to_axum!(endpoints::USER_MUTED_ROUTE) => muted::user_muted_handler, + endpoints::USER_ROUTE => view::user_view_handler, + endpoints::USER_DETAILS_ROUTE => details::user_details_handler, + endpoints::RELATIONSHIP_ROUTE => relationship::user_relationship_handler, + endpoints::USER_TAGS_ROUTE => tags::user_tags_handler, + endpoints::USER_TAGGERS_ROUTE => tags::user_taggers_handler, + endpoints::USER_COUNTS_ROUTE => counts::user_counts_handler, + endpoints::USER_FOLLOWERS_ROUTE => follows::user_followers_handler, + endpoints::USER_FOLLOWING_ROUTE => follows::user_following_handler, + endpoints::USER_FRIENDS_ROUTE => follows::user_friends_handler, + endpoints::USER_MUTED_ROUTE => muted::user_muted_handler, ) } From 2bd53cf060ace23d883f2cccad7804baa58033ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:52:06 +0100 Subject: [PATCH 07/42] deps: bump serde_json from 1.0.134 to 1.0.135 (#281) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.134 to 1.0.135. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.134...v1.0.135) --- updated-dependencies: - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7add41f0..de40ad4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3181,9 +3181,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 3f85865d..d6cabe04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ axum = "0.8.1" redis = { version = "0.27.6", features = ["tokio-comp", "json"] } neo4rs = "0.8.0" serde = { version = "1.0.217", features = ["derive"] } -serde_json = "1.0.134" +serde_json = "1.0.135" once_cell = "1.20.2" utoipa = { git = "https://github.com/juhaku/utoipa", rev = "d522f744259dc4fde5f45d187983fb68c8167029" } utoipa-swagger-ui = { git = "https://github.com/juhaku/utoipa", rev = "d522f744259dc4fde5f45d187983fb68c8167029", features = [ From cade79476e66af4a46f91d1b779caa298107f15f Mon Sep 17 00:00:00 2001 From: tipogi Date: Mon, 13 Jan 2025 11:25:26 +0100 Subject: [PATCH 08/42] error control in PUT events for RetryManager and add tag related integration tests --- Cargo.lock | 15 --- Cargo.toml | 1 - benches/watcher.rs | 9 +- examples/from_file.rs | 13 ++- src/db/kv/index/hash_map.rs | 50 ++++++++++ src/db/kv/index/mod.rs | 1 + src/db/kv/traits.rs | 42 +++++++- src/events/error.rs | 17 ++++ src/events/handlers/bookmark.rs | 4 +- src/events/handlers/follow.rs | 6 +- src/events/handlers/mute.rs | 5 +- src/events/handlers/post.rs | 15 ++- src/events/handlers/tag.rs | 24 +++-- src/events/handlers/user.rs | 6 +- src/events/mod.rs | 4 +- src/events/processor.rs | 92 +++++++++++------ src/events/retry.rs | 143 +++++++++++++++++++-------- src/models/tag/search.rs | 2 +- src/watcher.rs | 2 +- tests/watcher/posts/utils.rs | 10 +- tests/watcher/tags/fail_index.rs | 4 +- tests/watcher/tags/mod.rs | 2 + tests/watcher/tags/post_del.rs | 2 +- tests/watcher/tags/post_muti_user.rs | 40 ++++---- tests/watcher/tags/post_put.rs | 4 +- tests/watcher/tags/retry_post_tag.rs | 90 +++++++++++++++++ tests/watcher/tags/retry_user_tag.rs | 81 +++++++++++++++ tests/watcher/tags/utils.rs | 2 + tests/watcher/users/raw.rs | 1 + tests/watcher/users/utils.rs | 9 +- tests/watcher/utils/watcher.rs | 27 ++++- 31 files changed, 574 insertions(+), 149 deletions(-) create mode 100644 src/db/kv/index/hash_map.rs create mode 100644 src/events/error.rs create mode 100644 tests/watcher/tags/retry_post_tag.rs create mode 100644 tests/watcher/tags/retry_user_tag.rs diff --git a/Cargo.lock b/Cargo.lock index f1ad1fa6..de40ad4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -860,20 +860,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "deadpool" version = "0.9.5" @@ -2581,7 +2567,6 @@ dependencies = [ "chrono", "const_format", "criterion", - "dashmap", "dotenv", "env_logger", "httpc-test", diff --git a/Cargo.toml b/Cargo.toml index 8d1a891c..d6cabe04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,6 @@ reqwest = "0.12.9" base32 = "0.5.1" blake3 = "1.5.5" url = "2.5.4" -dashmap = "6.1.0" async-trait = "0.1.84" [dev-dependencies] diff --git a/benches/watcher.rs b/benches/watcher.rs index d5083b83..7e590bcf 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -6,6 +6,7 @@ use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::{ events::retry::{RetryManager, SenderChannel}, + types::PubkyId, EventProcessor, }; use setup::run_setup; @@ -35,7 +36,7 @@ async fn create_homeserver_with_events() -> (Testnet, String, SenderChannel) { let sender_clone = retry_manager.sender.clone(); // Create new asynchronous task to control the failed events tokio::spawn(async move { - retry_manager.exec().await; + let _ = retry_manager.exec().await; }); // Create and delete a user profile (as per your requirement) @@ -83,9 +84,13 @@ fn bench_create_delete_user(c: &mut Criterion) { let sender_clone = sender.clone(); // Clone the sender for each iteration let homeserver_url_clone = homeserver_url.clone(); async move { + // Create hardcoded homeserver pubkyId + let id = PubkyId::try_from("66h9hkdaud4ekkuummh3b4zhk68iggzirqbomyktfhq5s84jirno") + .unwrap(); + // Benchmark the event processor initialization and run let mut event_processor = - EventProcessor::test(homeserver_url_clone, sender_clone).await; + EventProcessor::test(homeserver_url_clone, id, sender_clone).await; event_processor.run().await.unwrap(); } }); diff --git a/examples/from_file.rs b/examples/from_file.rs index 8171b70b..3b86d592 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -1,8 +1,10 @@ use anyhow::Result; +use pubky_nexus::events::retry::RetryManager; use pubky_nexus::{setup, types::DynError, Config, EventProcessor}; use std::fs::File; use std::io::{self, BufRead}; use std::path::Path; +use tokio::sync::mpsc; // Create that file and add the file with that format // PUT homeserver_uri @@ -14,10 +16,17 @@ async fn main() -> Result<(), DynError> { let config = Config::from_env(); setup(&config).await; - let mut event_processor = EventProcessor::from_config(&config).await?; + let retry_manager = RetryManager::initialise(mpsc::channel(1024)); + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = retry_manager.sender.clone(); + // Create new asynchronous task to control the failed events + tokio::spawn(async move { + let _ = retry_manager.exec().await; + }); - let events = read_events_from_file().unwrap(); + let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; + let events = read_events_from_file().unwrap(); event_processor.process_event_lines(events).await?; Ok(()) diff --git a/src/db/kv/index/hash_map.rs b/src/db/kv/index/hash_map.rs new file mode 100644 index 00000000..0035d0df --- /dev/null +++ b/src/db/kv/index/hash_map.rs @@ -0,0 +1,50 @@ +use crate::db::connectors::redis::get_redis_conn; +use crate::types::DynError; +use redis::AsyncCommands; + +/// Inserts a value into a Redis hash map under the specified prefix, key, and field +/// * `prefix` - A string slice representing the prefix for the key. This is typically used to group related keys in Redis +/// * `key` - A string slice representing the main key in the hash map +/// * `field` - A string slice representing the field within the hash map where the value will be stored +/// * `value` - A `String` containing the value to be stored in the hash map +pub async fn put(prefix: &str, key: &str, field: &str, value: String) -> Result<(), DynError> { + let index_key = format!("{}:{}", prefix, key); + let mut redis_conn = get_redis_conn().await?; + // HSETNX sets the field only if it does not exist. Returns 1 (true) or 0 (false). + let _: bool = redis_conn.hset(&index_key, field, value).await?; + Ok(()) +} + +/// Retrieves a value from a Redis hash map using the specified prefix, key, and field +/// # Arguments +/// * `prefix` - A string slice representing the prefix for the key. This is used to group related keys in Redis +/// * `key` - A string slice representing the main key in the hash map +/// * `field` - A string slice representing the field within the hash map from which the value will be retrieved +pub async fn get(prefix: &str, key: &str, field: &str) -> Result, DynError> { + let index_key = format!("{}:{}", prefix, key); + let mut redis_conn = get_redis_conn().await?; + + // HGET retrieves the value for the given field. + let value: Option = redis_conn.hget(&index_key, field).await?; + Ok(value) +} + +/// Deletes one or more fields from a Redis hash map under the specified prefix and key. +/// # Arguments +/// * `prefix` - A string slice representing the prefix for the key. This is used to group related keys in Redis +/// * `key` - A string slice representing the main key in the hash map +/// * `fields` - A slice of string slices representing the fields to be removed from the hash map +pub async fn _del(prefix: &str, key: &str, fields: &[&str]) -> Result<(), DynError> { + if fields.is_empty() { + return Ok(()); + } + + let index_key = format!("{}:{}", prefix, key); + let mut redis_conn = get_redis_conn().await?; + + // The HDEL command is used to remove one or more fields from a hash. + // It returns the number of fields that were removed + let _removed_count: i32 = redis_conn.hdel(index_key, fields).await?; + + Ok(()) +} diff --git a/src/db/kv/index/mod.rs b/src/db/kv/index/mod.rs index 480a2796..c8474db5 100644 --- a/src/db/kv/index/mod.rs +++ b/src/db/kv/index/mod.rs @@ -1,3 +1,4 @@ +pub mod hash_map; /// Module for redis Indexing operations split into modules by Redis types pub mod json; pub mod lists; diff --git a/src/db/kv/traits.rs b/src/db/kv/traits.rs index 6eda4682..73950503 100644 --- a/src/db/kv/traits.rs +++ b/src/db/kv/traits.rs @@ -529,12 +529,14 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { /// * `key_parts` - A slice of string slices that represent the parts used to form the key under which the sorted set is stored. /// * `member` - A slice of string slices that represent the parts used to form the key identifying the member within the sorted set. async fn check_sorted_set_member( + prefix: Option<&str>, key_parts: &[&str], member: &[&str], ) -> Result, DynError> { + let prefix = prefix.unwrap_or(SORTED_PREFIX); let key = key_parts.join(":"); let member_key = member.join(":"); - sorted_sets::check_member(SORTED_PREFIX, &key, &member_key).await + sorted_sets::check_member(prefix, &key, &member_key).await } /// Adds elements to a Redis sorted set using the provided key parts. @@ -679,4 +681,42 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { let key = key_parts.join(":"); sorted_sets::get_lex_range("Sorted", &key, min, max, skip, limit).await } + + // ############################################################ + // ############ HASH MAP related functions #################### + // ############################################################ + + /// Inserts a value into a Redis hash map at the specified key and field. + /// # Arguments + /// * `prefix` - An optional string slice representing the prefix for the key + /// * `key_parts` - A slice of string slices representing parts of the key + /// * `field` - A string slice representing the field in the hash map where the value will be stored. + /// * `value` - A `String` containing the value to be stored in the hash map + async fn put_index_hash_map( + prefix: Option<&str>, + key_parts: &[&str], + field: &str, + value: String, + ) -> Result<(), DynError> { + let prefix = prefix.unwrap_or(SORTED_PREFIX); + let key = key_parts.join(":"); + // Store the elements in the Redis sorted set + hash_map::put(prefix, &key, field, value).await + } + + /// Retrieves a value from a Redis hash map at the specified key and field. + /// # Arguments + /// * `prefix` - An optional string slice representing the prefix for the key + /// * `key_parts` - A slice of string slices representing parts of the key + /// * `field` - A string slice representing the field in the hash map from which the value will be retrieved + async fn get_index_hash_map( + prefix: Option<&str>, + key_parts: &[&str], + field: &str, + ) -> Result, DynError> { + let prefix = prefix.unwrap_or(SORTED_PREFIX); + let key = key_parts.join(":"); + // Store the elements in the Redis sorted set + hash_map::get(prefix, &key, field).await + } } diff --git a/src/events/error.rs b/src/events/error.rs new file mode 100644 index 00000000..dac03bba --- /dev/null +++ b/src/events/error.rs @@ -0,0 +1,17 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum EventProcessorError { + #[error("The user could not be indexed in nexus")] + UserNotSync, + #[error("The event could not be indexed because some graph dependency is missing")] + MissingDependency { dependency: Vec }, + #[error("The event does not exist anymore in the homeserver")] + ContentNotFound { dependency: String }, + #[error("PubkyClient could not reach/resolve the homeserver")] + NotResolvedHomeserver, + #[error("The event could not be parsed from a line")] + InvalidEvent, + #[error("")] + PubkyClientError, +} diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 821961bf..13fe4ed6 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -1,5 +1,6 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; +use crate::events::error::EventProcessorError; use crate::events::uri::ParsedUri; use crate::models::post::Bookmark; use crate::models::user::UserCounts; @@ -41,7 +42,8 @@ pub async fn sync_put( OperationOutcome::Updated => true, // TODO: Should return an error that should be processed by RetryManager OperationOutcome::Pending => { - return Err("WATCHER: Missing some dependency to index the model".into()) + let dependency = vec![format!("pubky://{author_id}/pub/pubky.app/posts/{post_id}")]; + return Err(EventProcessorError::MissingDependency { dependency }.into()); } }; diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index 2ad88a2a..a93d3411 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -1,5 +1,6 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; +use crate::events::error::EventProcessorError; use crate::models::follow::{Followers, Following, Friends, UserFollows}; use crate::models::notification::Notification; use crate::models::user::UserCounts; @@ -23,10 +24,9 @@ pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), match Followers::put_to_graph(&follower_id, &followee_id).await? { // Do not duplicate the follow relationship OperationOutcome::Updated => return Ok(()), - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager OperationOutcome::Pending => { - return Err("WATCHER: Missing some dependency to index the model".into()) + let dependency = vec![format!("pubky://{followee_id}/pub/pubky.app/profile.json")]; + return Err(EventProcessorError::MissingDependency { dependency }.into()); } // The relationship did not exist, create all related indexes OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index cc33cb19..35563aa4 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -1,4 +1,5 @@ use crate::db::graph::exec::OperationOutcome; +use crate::events::error::EventProcessorError; use crate::models::user::Muted; use crate::types::DynError; use crate::types::PubkyId; @@ -19,9 +20,9 @@ pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro // (user_id)-[:MUTED]->(muted_id) match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), - // TODO: Should return an error that should be processed by RetryManager OperationOutcome::Pending => { - Err("WATCHER: Missing some dependency to index the model".into()) + let dependency = vec![format!("pubky://{muted_id}/pub/pubky.app/profile.json")]; + Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { // SAVE TO INDEX diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index d4480639..6e441932 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -1,5 +1,6 @@ use crate::db::graph::exec::{exec_single_row, execute_graph_operation, OperationOutcome}; use crate::db::kv::index::json::JsonAction; +use crate::events::error::EventProcessorError; use crate::events::uri::ParsedUri; use crate::models::notification::{Notification, PostChangedSource, PostChangedType}; use crate::models::post::{ @@ -41,10 +42,18 @@ pub async fn sync_put( let existed = match post_details.put_to_graph(&post_relationships).await? { OperationOutcome::CreatedOrDeleted => false, OperationOutcome::Updated => true, - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager OperationOutcome::Pending => { - return Err("WATCHER: Missing some dependency to index the model".into()) + let mut dependency = Vec::new(); + if let Some(replied_uri) = &post_relationships.replied { + dependency.push(replied_uri.clone()); + } + if let Some(reposted_uri) = &post_relationships.reposted { + dependency.push(reposted_uri.clone()); + } + if dependency.is_empty() { + dependency.push(format!("pubky://{author_id}/pub/pubky.app/profile.json")) + } + return Err(EventProcessorError::MissingDependency { dependency }.into()); } }; diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 39bde6fc..0badd38a 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -1,5 +1,6 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; +use crate::events::error::EventProcessorError; use crate::events::uri::ParsedUri; use crate::models::notification::Notification; use crate::models::post::{PostCounts, PostStream}; @@ -79,10 +80,9 @@ async fn put_sync_post( .await? { OperationOutcome::Updated => Ok(()), - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager OperationOutcome::Pending => { - Err("WATCHER: Missing some dependency to index the model".into()) + let dependency = vec![format!("pubky://{author_id}/pub/pubky.app/posts/{post_id}")]; + Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { // SAVE TO INDEXES @@ -158,10 +158,11 @@ async fn put_sync_user( .await? { OperationOutcome::Updated => Ok(()), - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager OperationOutcome::Pending => { - Err("WATCHER: Missing some dependency to index the model".into()) + let dependency = vec![format!( + "pubky://{tagged_user_id}/pub/pubky.app/profile.json" + )]; + Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { // SAVE TO INDEX @@ -189,6 +190,17 @@ async fn put_sync_user( Ok(()) } } + // TagUser::put_to_graph( + // &tagger_user_id, + // &tagged_user_id, + // None, + // &tag_id, + // &tag_label, + // indexed_at, + // ) + // .await?; + // let dependency = format!("pubky://{tagged_user_id}/pub/pubky.app/profile.json"); + // Err(EventProcessorError::MissingDependency { dependency }.into()) } pub async fn del(user_id: PubkyId, tag_id: String) -> Result<(), DynError> { diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index d2ee1b79..9cd0279f 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -1,4 +1,5 @@ use crate::db::graph::exec::{execute_graph_operation, OperationOutcome}; +use crate::events::error::EventProcessorError; use crate::models::user::UserSearch; use crate::models::{ traits::Collection, @@ -25,7 +26,10 @@ pub async fn sync_put(user: PubkyAppUser, user_id: PubkyId) -> Result<(), DynErr // Create UserDetails object let user_details = UserDetails::from_homeserver(user, &user_id).await?; // SAVE TO GRAPH - user_details.put_to_graph().await?; + match user_details.put_to_graph().await { + Ok(_) => (), + Err(_) => return Err(EventProcessorError::UserNotSync.into()), + } // SAVE TO INDEX let user_id = user_details.id.clone(); UserSearch::put_to_index(&[&user_details]).await?; diff --git a/src/events/mod.rs b/src/events/mod.rs index 0deeacfc..ebd2b78f 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -3,8 +3,10 @@ use crate::{ types::{DynError, PubkyId}, }; use log::{debug, error}; +use serde::{Deserialize, Serialize}; use uri::ParsedUri; +pub mod error; pub mod handlers; pub mod processor; pub mod retry; @@ -42,7 +44,7 @@ enum ResourceType { } // Look for the end pattern after the start index, or use the end of the string if not found -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum EventType { Put, Del, diff --git a/src/events/processor.rs b/src/events/processor.rs index e1082de3..53c8aef7 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,5 +1,4 @@ -use std::time::Duration; - +use super::error::EventProcessorError; use super::retry::SenderChannel; use super::retry::SenderMessage; use super::Event; @@ -9,26 +8,28 @@ use crate::types::PubkyId; use crate::{models::homeserver::Homeserver, Config}; use log::{debug, error, info}; use reqwest::Client; +use serde::Deserialize; +use serde::Serialize; pub struct EventProcessor { http_client: Client, pub homeserver: Homeserver, limit: u32, - max_retries: u64, pub sender: SenderChannel, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum EventErrorType { NotResolveHomeserver, PubkyClientError, + MissingDependency, + GraphError, } impl EventProcessor { pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; - let max_retries = config.max_retries; info!( "Initialized Event Processor for homeserver: {:?}", @@ -39,26 +40,35 @@ impl EventProcessor { http_client: Client::new(), homeserver, limit, - max_retries, sender: tx, }) } /// Creates a new `EventProcessor` instance for testing purposes. /// - /// Initializes an `EventProcessor` with a mock homeserver and a default configuration, - /// making it suitable for use in integration tests and benchmarking scenarios. + /// This function initializes an `EventProcessor` configured with: + /// - A mock homeserver constructed using the provided `homeserver_url` and `homeserver_pubky`. + /// - A default configuration, including an HTTP client, a limit of 1000 events, and a sender channel. + /// + /// It is designed for use in integration tests, benchmarking scenarios, or other test environments + /// where a controlled and predictable `EventProcessor` instance is required. /// /// # Parameters - /// - `homeserver_url`: The URL of the homeserver to be used in the test environment. - pub async fn test(homeserver_url: String, tx: SenderChannel) -> Self { - let id = PubkyId("test".to_string()); - let homeserver = Homeserver::new(id, homeserver_url).await.unwrap(); + /// - `homeserver_url`: A `String` representing the URL of the homeserver to be used in the test environment. + /// - `homeserver_pubky`: A `PubkyId` instance representing the unique identifier for the homeserver's public key. + /// - `tx`: A `SenderChannel` used to handle outgoing messages or events. + pub async fn test( + homeserver_url: String, + homeserver_pubky: PubkyId, + tx: SenderChannel, + ) -> Self { + let homeserver = Homeserver::new(homeserver_pubky, homeserver_url) + .await + .unwrap(); Self { http_client: Client::new(), homeserver, limit: 1000, - max_retries: 3, sender: tx, } } @@ -127,7 +137,7 @@ impl EventProcessor { }; if let Some(event) = event { debug!("Processing event: {:?}", event); - self.handle_event_with_retry(event).await?; + self.handle_event(event).await?; } } } @@ -136,29 +146,49 @@ impl EventProcessor { } // Generic retry on event handler - async fn handle_event_with_retry(&self, event: Event) -> Result<(), DynError> { + async fn handle_event(&self, event: Event) -> Result<(), DynError> { match event.clone().handle().await { Ok(_) => Ok(()), Err(e) => { error!("Error while handling event: {}", e); - error!("PROCESSOR: Sending the event to RetryManager... Missing node(s) and/or relationship(s) to execute PUT or DEL operation(s)"); - // Send the failed event to the retry manager to retry indexing - let mut error_type = None; + let retry_event = match e.downcast_ref::() { + Some(EventProcessorError::UserNotSync) => RetryEvent::new( + &event.uri, + &event.event_type, + None, + EventErrorType::GraphError, + ), + Some(EventProcessorError::MissingDependency { dependency }) => RetryEvent::new( + &event.uri, + &event.event_type, + Some(dependency.clone()), + EventErrorType::MissingDependency, + ), + // Other retry errors must be ignored + _ => return Ok(()), + }; - if e.to_string() == "Generic error: Could not resolve homeserver" { - error_type = Some(EventErrorType::NotResolveHomeserver); - } else if e.to_string().contains("error sending request for url") { - error_type = Some(EventErrorType::PubkyClientError); - } - // TODO: Another else if to catch the error of the graph - if let Some(error) = error_type { - let fail_event = - RetryEvent::new(&event.uri, &event.event_type, None, error); - let sender = self.sender.lock().await; - sender - .send(SenderMessage::Add(self.homeserver.id.clone(), fail_event)) - .await?; + let sender = self.sender.lock().await; + match sender + .send(SenderMessage::Add(self.homeserver.id.clone(), retry_event)) + .await + { + Ok(_) => { + info!("Message send succesfully from the channel"); + // TODO: Investigate non-blocking alternatives + // The current use of `tokio::time::sleep` is intended to handle a situation where tasks in other threads + // are not being tracked. This could potentially lead to issues with writing `RetryEvents` in certain cases. + // Considerations: + // - Determine if this delay is genuinely necessary. Testing may reveal that the `RetryManager` thread + // handles retries adequately while the watcher remains active, making this timer redundant. + // - If the delay is required, explore a more efficient solution that does not block the thread + // and ensures proper handling of tasks in other threads. + // + // For now, the sleep is deactivated + //tokio::time::sleep(Duration::from_millis(500)).await; + } + Err(e) => error!("Err, {:?}", e), } Ok(()) } diff --git a/src/events/retry.rs b/src/events/retry.rs index b27046ee..432f9e84 100644 --- a/src/events/retry.rs +++ b/src/events/retry.rs @@ -1,44 +1,112 @@ -use dashmap::DashMap; -use log::debug; -use std::{collections::LinkedList, sync::Arc}; +use chrono::Utc; +use log::info; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; use tokio::sync::{ mpsc::{Receiver, Sender}, Mutex, }; -use crate::types::PubkyId; +use crate::{ + types::{DynError, PubkyId}, + RedisOps, +}; use super::{processor::EventErrorType, EventType}; pub const CHANNEL_BUFFER: usize = 1024; +pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; +pub const RETRY_MANAGER_EVENTS_INDEX: &str = "events"; +pub const RETRY_MANAGER_STATE_INDEX: &str = "state"; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct RetryEvent { - pub uri: String, // URI of the resource - pub event_type: EventType, // Type of event (e.g., PUT, DEL) - pub timestamp: u64, // Unix timestamp when the event was received - pub dependency: Option, // Optional parent URI for dependency tracking - pub retry_count: u32, // Number of retries attempted - pub error_type: EventErrorType, // Optional field to track failure reasons + pub uri: String, // URI of the resource + pub event_type: EventType, // Type of event (e.g., PUT, DEL) + pub timestamp: i64, // Unix timestamp when the event was received + pub dependency: Option>, // Optional parent URI for dependency tracking + pub retry_count: u32, // Number of retries attempted + pub error_type: EventErrorType, // Optional field to track failure reasons } +impl RedisOps for RetryEvent {} + impl RetryEvent { pub fn new( uri: &String, event_type: &EventType, - dependency: Option, + dependency: Option>, error_type: EventErrorType, ) -> Self { Self { uri: uri.to_string(), event_type: event_type.clone(), - // TODO: Add now unixtime - timestamp: 1733831902, + timestamp: Utc::now().timestamp_millis(), dependency, retry_count: 0, error_type, } } + + fn get_events_index(homeserver: &str) -> [&str; 2] { + [homeserver, RETRY_MANAGER_EVENTS_INDEX] + } + + fn get_state_index(homeserver: &str) -> [&str; 2] { + [homeserver, RETRY_MANAGER_STATE_INDEX] + } + + pub async fn put_to_index(&self, homeserver: &str) -> Result<(), DynError> { + Self::put_index_sorted_set( + &Self::get_events_index(homeserver), + &[(self.timestamp as f64, &self.uri)], + Some(RETRY_MAMAGER_PREFIX), + None, + ) + .await?; + + let event_serialized = serde_json::to_string(self)?; + + Self::put_index_hash_map( + Some(RETRY_MAMAGER_PREFIX), + &Self::get_state_index(homeserver), + &self.uri, + event_serialized, + ) + .await?; + Ok(()) + } + + pub async fn check_uri(homeserver: &str, pubky_uri: &str) -> Result, DynError> { + if let Some(post_details) = Self::check_sorted_set_member( + Some(RETRY_MAMAGER_PREFIX), + &Self::get_events_index(homeserver), + &[pubky_uri], + ) + .await? + { + return Ok(Some(post_details)); + } + Ok(None) + } + + pub async fn get_from_hash_map_index( + homeserver: &str, + pubky_uri: &str, + ) -> Result, DynError> { + let mut found_event = None; + if let Some(event_state) = Self::get_index_hash_map( + Some(RETRY_MAMAGER_PREFIX), + &Self::get_state_index(homeserver), + pubky_uri, + ) + .await? + { + let event = serde_json::from_str::(&event_state)?; + found_event = Some(event); + } + Ok(found_event) + } } #[derive(Debug, Clone)] @@ -53,7 +121,6 @@ type ReceiverChannel = Arc>>; pub struct RetryManager { pub sender: SenderChannel, receiver: ReceiverChannel, - fail_events: DashMap>, } /// Initializes a new `RetryManager` with a message-passing channel. @@ -70,51 +137,41 @@ impl RetryManager { Self { receiver: Arc::new(Mutex::new(rx)), sender: Arc::new(Mutex::new(tx)), - fail_events: DashMap::new(), } } - pub async fn exec(&self) { + pub async fn exec(&self) -> Result<(), DynError> { let mut rx = self.receiver.lock().await; + // Listen all the messages in the channel while let Some(message) = rx.recv().await { match message { SenderMessage::Retry(homeserver_pubky) => { - self.retry_events_for_homeserver(&homeserver_pubky).await; + self.retry_events_for_homeserver(&homeserver_pubky); } SenderMessage::Add(homeserver_pubky, retry_event) => { - self.add_fail_event(homeserver_pubky, retry_event); + self.add_fail_event(homeserver_pubky, retry_event).await?; } } } + Ok(()) } - async fn retry_events_for_homeserver(&self, homeserver_pubky: &str) { - if let Some(retry_events) = self.fail_events.get(homeserver_pubky) { - debug!( - "Trying to fetch again the failing events from {:?}...", - homeserver_pubky - ); - for event in retry_events.iter() { - debug!( - "-> {:?}:{}:{:?}", - event.event_type, event.uri, event.error_type - ); - } - } else { - debug!("No retry events found for key: {}", homeserver_pubky); - } + fn retry_events_for_homeserver(&self, _homeserver_pubky: &str) { + // WIP: Retrieve the homeserver events from the SORTED SET + // RetryManager:homeserver_pubky:events } - fn add_fail_event(&self, homeserver_pubky: PubkyId, retry_event: RetryEvent) { - debug!( - "Added fail event in the HashMap: {:?}: {}", + async fn add_fail_event( + &self, + homeserver_pubky: PubkyId, + retry_event: RetryEvent, + ) -> Result<(), DynError> { + info!( + "Add fail event to Redis: {:?}: {}", retry_event.event_type, retry_event.uri ); - // Write the event in the HashMap - let mut list = self - .fail_events - .entry(homeserver_pubky.to_string()) - .or_default(); - list.push_back(retry_event); + // Write in the index + retry_event.put_to_index(&homeserver_pubky).await?; + Ok(()) } } diff --git a/src/models/tag/search.rs b/src/models/tag/search.rs index 6128032a..c39b7e12 100644 --- a/src/models/tag/search.rs +++ b/src/models/tag/search.rs @@ -129,7 +129,7 @@ impl TagSearch { ) -> Result<(), DynError> { let post_key_slice: &[&str] = &[author_id, post_id]; let key_parts = [&TAG_GLOBAL_POST_TIMELINE[..], &[tag_label]].concat(); - let tag_search = Self::check_sorted_set_member(&key_parts, post_key_slice).await?; + let tag_search = Self::check_sorted_set_member(None, &key_parts, post_key_slice).await?; if tag_search.is_none() { let option = PostDetails::try_from_index_json(post_key_slice).await?; if let Some(post_details) = option { diff --git a/src/watcher.rs b/src/watcher.rs index 789b4b41..1f25466d 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -26,7 +26,7 @@ async fn main() -> Result<(), Box> { let sender_clone = retry_manager.sender.clone(); // Create new asynchronous task to control the failed events tokio::spawn(async move { - retry_manager.exec().await; + let _ = retry_manager.exec().await; }); // Create and configure the event processor diff --git a/tests/watcher/posts/utils.rs b/tests/watcher/posts/utils.rs index 1b48ebe9..27ecf839 100644 --- a/tests/watcher/posts/utils.rs +++ b/tests/watcher/posts/utils.rs @@ -40,7 +40,7 @@ pub async fn check_member_global_timeline_user_post( ) -> Result> { let post_key: &[&str] = &[user_id, post_id]; let global_timeline_timestamp = - PostStream::check_sorted_set_member(&POST_TIMELINE_KEY_PARTS, post_key) + PostStream::check_sorted_set_member(None, &POST_TIMELINE_KEY_PARTS, post_key) .await .unwrap(); Ok(global_timeline_timestamp) @@ -52,7 +52,7 @@ pub async fn check_member_user_post_timeline( ) -> Result> { let post_stream_key_parts = [&POST_PER_USER_KEY_PARTS[..], &[user_id]].concat(); let post_timeline_timestamp = - PostStream::check_sorted_set_member(&post_stream_key_parts, &[post_id]) + PostStream::check_sorted_set_member(None, &post_stream_key_parts, &[post_id]) .await .unwrap(); Ok(post_timeline_timestamp) @@ -64,7 +64,7 @@ pub async fn check_member_user_replies_timeline( ) -> Result> { let post_stream_key_parts = [&POST_REPLIES_PER_USER_KEY_PARTS[..], &[user_id]].concat(); let post_timeline_timestamp = - PostStream::check_sorted_set_member(&post_stream_key_parts, &[post_id]) + PostStream::check_sorted_set_member(None, &post_stream_key_parts, &[post_id]) .await .unwrap(); Ok(post_timeline_timestamp) @@ -72,7 +72,7 @@ pub async fn check_member_user_replies_timeline( pub async fn check_member_total_engagement_user_posts(post_key: &[&str]) -> Result> { let total_engagement = - PostStream::check_sorted_set_member(&POST_TOTAL_ENGAGEMENT_KEY_PARTS, post_key) + PostStream::check_sorted_set_member(None, &POST_TOTAL_ENGAGEMENT_KEY_PARTS, post_key) .await .unwrap(); Ok(total_engagement) @@ -85,7 +85,7 @@ pub async fn check_member_post_replies( ) -> Result> { let key_parts = [&POST_REPLIES_PER_POST_KEY_PARTS[..], &[author_id, post_id]].concat(); - let post_replies = PostStream::check_sorted_set_member(&key_parts, post_key) + let post_replies = PostStream::check_sorted_set_member(None, &key_parts, post_key) .await .unwrap(); Ok(post_replies) diff --git a/tests/watcher/tags/fail_index.rs b/tests/watcher/tags/fail_index.rs index c7a6c318..829d5742 100644 --- a/tests/watcher/tags/fail_index.rs +++ b/tests/watcher/tags/fail_index.rs @@ -5,8 +5,10 @@ use log::error; use pubky_app_specs::{traits::HashId, PubkyAppPost, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; +// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), +// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] -async fn test_homeserver_tag_user_not_found() -> Result<()> { +async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { let mut test = WatcherTest::setup().await?; let tagged_keypair = Keypair::random(); diff --git a/tests/watcher/tags/mod.rs b/tests/watcher/tags/mod.rs index 38cb8bd4..c38aa3e5 100644 --- a/tests/watcher/tags/mod.rs +++ b/tests/watcher/tags/mod.rs @@ -3,6 +3,8 @@ mod post_del; mod post_muti_user; mod post_notification; mod post_put; +mod retry_post_tag; +mod retry_user_tag; mod user_notification; mod user_to_self_put; mod user_to_user_del; diff --git a/tests/watcher/tags/post_del.rs b/tests/watcher/tags/post_del.rs index 96b5b6af..d08cbea8 100644 --- a/tests/watcher/tags/post_del.rs +++ b/tests/watcher/tags/post_del.rs @@ -144,7 +144,7 @@ async fn test_homeserver_del_tag_post() -> Result<()> { assert!(tag_timeline.is_none()); // Assert hot tag score: Sorted:Post:Global:Hot:label - let total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label]) + let total_engagement = Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label]) .await .unwrap() .unwrap(); diff --git a/tests/watcher/tags/post_muti_user.rs b/tests/watcher/tags/post_muti_user.rs index f5a43a9e..8500a821 100644 --- a/tests/watcher/tags/post_muti_user.rs +++ b/tests/watcher/tags/post_muti_user.rs @@ -66,12 +66,12 @@ async fn test_homeserver_multi_user() -> Result<()> { // Avoid errors, if the score does not exist. Using that variable in the last assert of the test let actual_water_tag_hot_score = - Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_water]) + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_water]) .await .unwrap() .unwrap_or_default(); let actual_fire_tag_hot_score = - Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_fire]) + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_fire]) .await .unwrap() .unwrap_or_default(); @@ -217,15 +217,17 @@ async fn test_homeserver_multi_user() -> Result<()> { assert_eq!(total_engagement.unwrap(), 5); // Assert hot tag score: Sorted:Post:Global:Hot:label - let water_total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_water]) - .await - .unwrap() - .unwrap(); + let water_total_engagement = + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_water]) + .await + .unwrap() + .unwrap(); assert_eq!(water_total_engagement, actual_water_tag_hot_score + 3); - let fire_total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_fire]) - .await - .unwrap() - .unwrap(); + let fire_total_engagement = + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_fire]) + .await + .unwrap() + .unwrap(); assert_eq!(fire_total_engagement, actual_fire_tag_hot_score + 2); // Step 4: DEL tag from homeserver @@ -318,15 +320,17 @@ async fn test_homeserver_multi_user() -> Result<()> { } // Assert hot tag score: Sorted:Post:Global:Hot:label - let water_total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_water]) - .await - .unwrap() - .unwrap(); + let water_total_engagement = + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_water]) + .await + .unwrap() + .unwrap(); assert_eq!(water_total_engagement, actual_water_tag_hot_score); - let fire_total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label_fire]) - .await - .unwrap() - .unwrap(); + let fire_total_engagement = + Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label_fire]) + .await + .unwrap() + .unwrap(); assert_eq!(fire_total_engagement, actual_fire_tag_hot_score); let notifications = Notification::get_by_id(author_id, Pagination::default()) diff --git a/tests/watcher/tags/post_put.rs b/tests/watcher/tags/post_put.rs index 95ba3c46..46bc70ba 100644 --- a/tests/watcher/tags/post_put.rs +++ b/tests/watcher/tags/post_put.rs @@ -57,7 +57,7 @@ async fn test_homeserver_put_tag_post() -> Result<()> { ); // Avoid errors, if the score does not exist. Using that variable in the last assert of the test - let actual_tag_hot_score = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label]) + let actual_tag_hot_score = Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label]) .await .unwrap() .unwrap_or_default(); @@ -156,7 +156,7 @@ async fn test_homeserver_put_tag_post() -> Result<()> { ); // Assert hot tag score: Sorted:Post:Global:Hot:label - let total_engagement = Taggers::check_sorted_set_member(&TAG_GLOBAL_HOT, &[label]) + let total_engagement = Taggers::check_sorted_set_member(None, &TAG_GLOBAL_HOT, &[label]) .await .unwrap() .unwrap(); diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs new file mode 100644 index 00000000..20635b67 --- /dev/null +++ b/tests/watcher/tags/retry_post_tag.rs @@ -0,0 +1,90 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use chrono::Utc; +use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{processor::EventErrorType, retry::RetryEvent, EventType}; + +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let tagger_keypair = Keypair::random(); + let tagger_user = PubkyAppUser { + bio: Some("test_homeserver_user_tag_event_to_queue".to_string()), + image: None, + links: None, + name: "Watcher:Retry:Post:CannotTag:Tagger:Sync".to_string(), + status: None, + }; + let tagger_user_id = test.create_user(&tagger_keypair, &tagger_user).await?; + + // Create a key but it would not be synchronised in nexus + let author_keypair = Keypair::random(); + let author = PubkyAppUser { + bio: Some("test_homeserver_user_tag_event_to_queue".to_string()), + image: None, + links: None, + name: "Watcher:Retry:Post:CannotTag:Author:Sync".to_string(), + status: None, + }; + let author_id = test.create_user(&author_keypair, &author).await?; + + // => Create user tag + let label = "peak_limit"; + + // Create a tag in a fake post + let tag = PubkyAppTag { + uri: format!("pubky://{author_id}/pub/pubky.app/posts/{}", "0032Q4SFBFDG"), + label: label.to_string(), + created_at: Utc::now().timestamp_millis(), + }; + + let tag_blob = serde_json::to_vec(&tag)?; + let tag_url = format!( + "pubky://{tagger_user_id}/pub/pubky.app/tags/{}", + tag.create_id() + ); + + // PUT user tag + // That operation is going to write the event in the pending events queue, so block a bit the thread + // to let write the indexes + test.put(tag_url.as_str(), tag_blob.clone()).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let homeserver_pubky = test.get_homeserver_pubky(); + + // Assert if the event is in the timeline + let timeline = RetryEvent::check_uri(&homeserver_pubky, &tag_url) + .await + .unwrap(); + assert!(timeline.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_hash_map_index(&homeserver_pubky, &tag_url) + .await + .unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.uri, tag_url); + assert_eq!(event_state.event_type, EventType::Put); + assert_eq!(event_state.error_type, EventErrorType::MissingDependency); + assert_eq!(event_state.retry_count, 0); + + let dependency = format!( + "pubky://{author_id}/pub/pubky.app/posts/{}", + "0032Q4SFBFD4G" + ); + assert!(event_state.dependency.is_some()); + let dependency_list = event_state.dependency.unwrap(); + assert_eq!(dependency_list.len(), 1); + assert_eq!(dependency_list[0], dependency); + + Ok(()) +} diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs new file mode 100644 index 00000000..b6192b05 --- /dev/null +++ b/tests/watcher/tags/retry_user_tag.rs @@ -0,0 +1,81 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use chrono::Utc; +use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{processor::EventErrorType, retry::RetryEvent, EventType}; + +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let tagger_keypair = Keypair::random(); + let tagger_user = PubkyAppUser { + bio: Some("test_homeserver_user_tag_event_to_queue".to_string()), + image: None, + links: None, + name: "Watcher:Retry:User:CannotTag:Tagger:Sync".to_string(), + status: None, + }; + let tagger_user_id = test.create_user(&tagger_keypair, &tagger_user).await?; + + // Create a user key but it would not be synchronised in nexus + let shadow_keypair = Keypair::random(); + test.register_user(&shadow_keypair).await?; + let shadow_user_id = shadow_keypair.public_key().to_z32(); + + // => Create user tag + let label = "friendly"; + + let tag = PubkyAppTag { + uri: format!("pubky://{}/pub/pubky.app/profile.json", shadow_user_id), + label: label.to_string(), + created_at: Utc::now().timestamp_millis(), + }; + + let tag_blob = serde_json::to_vec(&tag)?; + let tag_url = format!( + "pubky://{}/pub/pubky.app/tags/{}", + tagger_user_id, + tag.create_id() + ); + + // PUT user tag + // That operation is going to write the event in the pending events queue, so block a bit the thread + // to let write the indexes + test.put(tag_url.as_str(), tag_blob.clone()).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let homeserver_pubky = test.get_homeserver_pubky(); + + // Assert if the event is in the timeline + let timeline = RetryEvent::check_uri(&homeserver_pubky, &tag_url) + .await + .unwrap(); + assert!(timeline.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_hash_map_index(&homeserver_pubky, &tag_url) + .await + .unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.uri, tag_url); + assert_eq!(event_state.event_type, EventType::Put); + assert_eq!(event_state.error_type, EventErrorType::MissingDependency); + assert_eq!(event_state.retry_count, 0); + + let dependency = format!("pubky://{shadow_user_id}/pub/pubky.app/profile.json"); + assert!(event_state.dependency.is_some()); + let dependency_list = event_state.dependency.unwrap(); + assert_eq!(dependency_list.len(), 1); + assert_eq!(dependency_list[0], dependency); + + Ok(()) +} diff --git a/tests/watcher/tags/utils.rs b/tests/watcher/tags/utils.rs index 6ebb9e3e..0dfc682e 100644 --- a/tests/watcher/tags/utils.rs +++ b/tests/watcher/tags/utils.rs @@ -63,6 +63,7 @@ pub async fn check_member_total_engagement_post_tag( label: &str, ) -> Result> { let total_engagement = TagSearch::check_sorted_set_member( + None, &[&TAG_GLOBAL_POST_ENGAGEMENT[..], &[label]].concat(), post_key, ) @@ -76,6 +77,7 @@ pub async fn check_member_post_tag_global_timeline( label: &str, ) -> Result> { let exist_in_timeline = TagSearch::check_sorted_set_member( + None, &[&TAG_GLOBAL_POST_TIMELINE[..], &[label]].concat(), post_key, ) diff --git a/tests/watcher/users/raw.rs b/tests/watcher/users/raw.rs index 66be1874..2de3a306 100644 --- a/tests/watcher/users/raw.rs +++ b/tests/watcher/users/raw.rs @@ -65,6 +65,7 @@ async fn test_homeserver_user_put_event() -> Result<()> { // Sorted:Users:Name let is_member = UserSearch::check_sorted_set_member( + None, &USER_NAME_KEY_PARTS, &[&user.name.to_lowercase(), &user_id], ) diff --git a/tests/watcher/users/utils.rs b/tests/watcher/users/utils.rs index 2de7d68f..d7249abf 100644 --- a/tests/watcher/users/utils.rs +++ b/tests/watcher/users/utils.rs @@ -9,16 +9,17 @@ use pubky_nexus::{ pub async fn check_member_most_followed(user_id: &str) -> Result> { let pioneer_score = - UserStream::check_sorted_set_member(&USER_MOSTFOLLOWED_KEY_PARTS, &[user_id]) + UserStream::check_sorted_set_member(None, &USER_MOSTFOLLOWED_KEY_PARTS, &[user_id]) .await .unwrap(); Ok(pioneer_score) } pub async fn check_member_user_pioneer(user_id: &str) -> Result> { - let pioneer_score = UserStream::check_sorted_set_member(&USER_PIONEERS_KEY_PARTS, &[user_id]) - .await - .unwrap(); + let pioneer_score = + UserStream::check_sorted_set_member(None, &USER_PIONEERS_KEY_PARTS, &[user_id]) + .await + .unwrap(); Ok(pioneer_score) } diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index ac66f12e..54784ee9 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -7,9 +7,12 @@ use pubky_app_specs::{ }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; +use pubky_nexus::events::{ + retry::{RetryManager, CHANNEL_BUFFER}, + Event, +}; +use pubky_nexus::types::{DynError, PubkyId}; use pubky_nexus::{setup, Config, EventProcessor, PubkyConnector}; -use pubky_nexus::events::{Event, retry::{RetryManager, CHANNEL_BUFFER}}; -use pubky_nexus::types::DynError; use serde_json::to_vec; use tokio::sync::mpsc; @@ -51,7 +54,7 @@ impl WatcherTest { let sender_clone = retry_manager.sender.clone(); tokio::spawn(async move { - retry_manager.exec().await; + let _ = retry_manager.exec().await; }); match PubkyConnector::initialise(&config, Some(&testnet)) { @@ -59,7 +62,15 @@ impl WatcherTest { Err(e) => debug!("WatcherTest: {}", e), } - let event_processor = EventProcessor::test(homeserver_url, sender_clone).await; + let homeserver_pubky = homeserver.public_key().to_uri_string(); + // Slice after the 3rd character + let pubky_part = &homeserver_pubky["pk:".len()..]; + // Not save, + let homeserver_pubky = + PubkyId::try_from(&pubky_part).expect("PubkyId: Cannot get the homeserver public key"); + + let event_processor = + EventProcessor::test(homeserver_url, homeserver_pubky, sender_clone).await; Ok(Self { config, @@ -84,6 +95,14 @@ impl WatcherTest { Ok(()) } + pub fn get_homeserver_pubky(&self) -> PubkyId { + let homeserver_pubky = self.homeserver.public_key().to_uri_string(); + // Slice after the 3rd character + let pubky_part = &homeserver_pubky["pk:".len()..]; + // Not save, + PubkyId::try_from(&pubky_part).expect("PubkyId: Cannot get the homeserver public key") + } + /// Sends a PUT request to the homeserver with the provided blob of data. /// /// This function performs the following steps: From bd6eeb237564844e9e0ffdc13f1d75d7b710a2cc Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 14 Jan 2025 06:24:57 +0100 Subject: [PATCH 09/42] refactor RetryEvent and crete new folder for RetryManager --- benches/watcher.rs | 2 +- examples/from_file.rs | 2 +- src/events/error.rs | 3 +- src/events/handlers/tag.rs | 11 -- src/events/mod.rs | 12 ++ src/events/processor.rs | 28 ++--- src/events/retry.rs | 177 --------------------------- src/events/retry/event.rs | 96 +++++++++++++++ src/events/retry/manager.rs | 91 ++++++++++++++ src/events/retry/mod.rs | 2 + src/watcher.rs | 5 +- tests/watcher/tags/retry_post_tag.rs | 44 +++---- tests/watcher/tags/retry_user_tag.rs | 34 ++--- tests/watcher/utils/watcher.rs | 8 +- 14 files changed, 262 insertions(+), 253 deletions(-) delete mode 100644 src/events/retry.rs create mode 100644 src/events/retry/event.rs create mode 100644 src/events/retry/manager.rs create mode 100644 src/events/retry/mod.rs diff --git a/benches/watcher.rs b/benches/watcher.rs index 7e590bcf..160ba2fa 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -5,7 +5,7 @@ use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::{ - events::retry::{RetryManager, SenderChannel}, + events::manager::{RetryManager, SenderChannel}, types::PubkyId, EventProcessor, }; diff --git a/examples/from_file.rs b/examples/from_file.rs index 3b86d592..0caebd58 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use pubky_nexus::events::retry::RetryManager; +use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::{setup, types::DynError, Config, EventProcessor}; use std::fs::File; use std::io::{self, BufRead}; diff --git a/src/events/error.rs b/src/events/error.rs index dac03bba..4d5a9837 100644 --- a/src/events/error.rs +++ b/src/events/error.rs @@ -1,6 +1,7 @@ +use serde::{Deserialize, Serialize}; use thiserror::Error; -#[derive(Error, Debug)] +#[derive(Error, Debug, Clone, Serialize, Deserialize)] pub enum EventProcessorError { #[error("The user could not be indexed in nexus")] UserNotSync, diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 0badd38a..728fd533 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -190,17 +190,6 @@ async fn put_sync_user( Ok(()) } } - // TagUser::put_to_graph( - // &tagger_user_id, - // &tagged_user_id, - // None, - // &tag_id, - // &tag_label, - // indexed_at, - // ) - // .await?; - // let dependency = format!("pubky://{tagged_user_id}/pub/pubky.app/profile.json"); - // Err(EventProcessorError::MissingDependency { dependency }.into()) } pub async fn del(user_id: PubkyId, tag_id: String) -> Result<(), DynError> { diff --git a/src/events/mod.rs b/src/events/mod.rs index ebd2b78f..45924623 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -1,3 +1,5 @@ +use std::fmt; + use crate::{ db::connectors::pubky::PubkyConnector, types::{DynError, PubkyId}, @@ -50,6 +52,16 @@ pub enum EventType { Del, } +impl fmt::Display for EventType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let upper_case_str = match self { + EventType::Put => "PUT", + EventType::Del => "DEL", + }; + write!(f, "{}", upper_case_str) + } +} + #[derive(Debug, Clone)] pub struct Event { uri: String, diff --git a/src/events/processor.rs b/src/events/processor.rs index 53c8aef7..d4438fec 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,8 +1,7 @@ use super::error::EventProcessorError; -use super::retry::SenderChannel; -use super::retry::SenderMessage; +use super::retry::manager::{SenderChannel, SenderMessage}; use super::Event; -use crate::events::retry::RetryEvent; +use crate::events::retry::event::RetryEvent; use crate::types::DynError; use crate::types::PubkyId; use crate::{models::homeserver::Homeserver, Config}; @@ -153,25 +152,22 @@ impl EventProcessor { error!("Error while handling event: {}", e); let retry_event = match e.downcast_ref::() { - Some(EventProcessorError::UserNotSync) => RetryEvent::new( - &event.uri, - &event.event_type, - None, - EventErrorType::GraphError, - ), - Some(EventProcessorError::MissingDependency { dependency }) => RetryEvent::new( - &event.uri, - &event.event_type, - Some(dependency.clone()), - EventErrorType::MissingDependency, - ), + Some(event_processor_error) => RetryEvent::new(event_processor_error.clone()), // Other retry errors must be ignored _ => return Ok(()), }; + // Generate a compress index to save in the cache + let index = if let Some(retry_index) = RetryEvent::generate_index_key(&event.uri) { + retry_index + } else { + // This block is unlikely to be reached, as it would typically fail during the validation process. + return Ok(()); + }; + let index_key = format!("{} {}", event.event_type, index); let sender = self.sender.lock().await; match sender - .send(SenderMessage::Add(self.homeserver.id.clone(), retry_event)) + .send(SenderMessage::Add(index_key, retry_event)) .await { Ok(_) => { diff --git a/src/events/retry.rs b/src/events/retry.rs deleted file mode 100644 index 432f9e84..00000000 --- a/src/events/retry.rs +++ /dev/null @@ -1,177 +0,0 @@ -use chrono::Utc; -use log::info; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use tokio::sync::{ - mpsc::{Receiver, Sender}, - Mutex, -}; - -use crate::{ - types::{DynError, PubkyId}, - RedisOps, -}; - -use super::{processor::EventErrorType, EventType}; - -pub const CHANNEL_BUFFER: usize = 1024; -pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; -pub const RETRY_MANAGER_EVENTS_INDEX: &str = "events"; -pub const RETRY_MANAGER_STATE_INDEX: &str = "state"; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RetryEvent { - pub uri: String, // URI of the resource - pub event_type: EventType, // Type of event (e.g., PUT, DEL) - pub timestamp: i64, // Unix timestamp when the event was received - pub dependency: Option>, // Optional parent URI for dependency tracking - pub retry_count: u32, // Number of retries attempted - pub error_type: EventErrorType, // Optional field to track failure reasons -} - -impl RedisOps for RetryEvent {} - -impl RetryEvent { - pub fn new( - uri: &String, - event_type: &EventType, - dependency: Option>, - error_type: EventErrorType, - ) -> Self { - Self { - uri: uri.to_string(), - event_type: event_type.clone(), - timestamp: Utc::now().timestamp_millis(), - dependency, - retry_count: 0, - error_type, - } - } - - fn get_events_index(homeserver: &str) -> [&str; 2] { - [homeserver, RETRY_MANAGER_EVENTS_INDEX] - } - - fn get_state_index(homeserver: &str) -> [&str; 2] { - [homeserver, RETRY_MANAGER_STATE_INDEX] - } - - pub async fn put_to_index(&self, homeserver: &str) -> Result<(), DynError> { - Self::put_index_sorted_set( - &Self::get_events_index(homeserver), - &[(self.timestamp as f64, &self.uri)], - Some(RETRY_MAMAGER_PREFIX), - None, - ) - .await?; - - let event_serialized = serde_json::to_string(self)?; - - Self::put_index_hash_map( - Some(RETRY_MAMAGER_PREFIX), - &Self::get_state_index(homeserver), - &self.uri, - event_serialized, - ) - .await?; - Ok(()) - } - - pub async fn check_uri(homeserver: &str, pubky_uri: &str) -> Result, DynError> { - if let Some(post_details) = Self::check_sorted_set_member( - Some(RETRY_MAMAGER_PREFIX), - &Self::get_events_index(homeserver), - &[pubky_uri], - ) - .await? - { - return Ok(Some(post_details)); - } - Ok(None) - } - - pub async fn get_from_hash_map_index( - homeserver: &str, - pubky_uri: &str, - ) -> Result, DynError> { - let mut found_event = None; - if let Some(event_state) = Self::get_index_hash_map( - Some(RETRY_MAMAGER_PREFIX), - &Self::get_state_index(homeserver), - pubky_uri, - ) - .await? - { - let event = serde_json::from_str::(&event_state)?; - found_event = Some(event); - } - Ok(found_event) - } -} - -#[derive(Debug, Clone)] -pub enum SenderMessage { - Retry(String), // Retry events associated with this key - Add(PubkyId, RetryEvent), // Add a new RetryEvent to the fail_events -} - -pub type SenderChannel = Arc>>; -type ReceiverChannel = Arc>>; - -pub struct RetryManager { - pub sender: SenderChannel, - receiver: ReceiverChannel, -} - -/// Initializes a new `RetryManager` with a message-passing channel. -/// -/// This function sets up the `RetryManager` by taking a tuple containing -/// a `Sender` and `Receiver` from a multi-producer, single-consumer (mpsc) channel. -/// -/// # Parameters -/// - `(tx, rx)`: A tuple containing: -/// - `tx` (`Sender`): The sending half of the mpsc channel, used to dispatch messages to the manager. -/// - `rx` (`Receiver`): The receiving half of the mpsc channel, used to listen for incoming messages. -impl RetryManager { - pub fn initialise((tx, rx): (Sender, Receiver)) -> Self { - Self { - receiver: Arc::new(Mutex::new(rx)), - sender: Arc::new(Mutex::new(tx)), - } - } - - pub async fn exec(&self) -> Result<(), DynError> { - let mut rx = self.receiver.lock().await; - // Listen all the messages in the channel - while let Some(message) = rx.recv().await { - match message { - SenderMessage::Retry(homeserver_pubky) => { - self.retry_events_for_homeserver(&homeserver_pubky); - } - SenderMessage::Add(homeserver_pubky, retry_event) => { - self.add_fail_event(homeserver_pubky, retry_event).await?; - } - } - } - Ok(()) - } - - fn retry_events_for_homeserver(&self, _homeserver_pubky: &str) { - // WIP: Retrieve the homeserver events from the SORTED SET - // RetryManager:homeserver_pubky:events - } - - async fn add_fail_event( - &self, - homeserver_pubky: PubkyId, - retry_event: RetryEvent, - ) -> Result<(), DynError> { - info!( - "Add fail event to Redis: {:?}: {}", - retry_event.event_type, retry_event.uri - ); - // Write in the index - retry_event.put_to_index(&homeserver_pubky).await?; - Ok(()) - } -} diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs new file mode 100644 index 00000000..47f9cf57 --- /dev/null +++ b/src/events/retry/event.rs @@ -0,0 +1,96 @@ +use chrono::Utc; +use serde::{Deserialize, Serialize}; + +use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; + +pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; +pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; +pub const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; +pub const HOMESERVER_PUBLIC_REPOSITORY: &str = "pub"; +pub const HOMESERVER_APP_REPOSITORY: &str = "pubky.app"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetryEvent { + // Number of retries attempted + pub retry_count: u32, + // This determines how the event should be processed during the retry process + pub error_type: EventProcessorError, +} + +impl RedisOps for RetryEvent {} + +impl RetryEvent { + pub fn new(error_type: EventProcessorError) -> Self { + Self { + retry_count: 0, + error_type, + } + } + + /// It processes a homeserver URI and extracts specific components to form a index key + /// in the format `"{pubkyId}:{repository_model}/{event_id}"` + /// # Parameters + /// - `event_uri`: A string slice representing the event URI to be processed + pub fn generate_index_key(event_uri: &str) -> Option { + let parts: Vec<&str> = event_uri.split('/').collect(); + if parts.len() >= 7 + && parts[0] == "pubky:" + && parts[3] == HOMESERVER_PUBLIC_REPOSITORY + && parts[4] == HOMESERVER_APP_REPOSITORY + { + Some(format!("{}:{}/{}", parts[2], parts[5], parts[6])) + } else { + None + } + } + + pub async fn put_to_index(&self, event_line: String) -> Result<(), DynError> { + Self::put_index_sorted_set( + &RETRY_MANAGER_EVENTS_INDEX, + // NOTE: Don't know if we should use now timestamp or the event timestamp + &[(Utc::now().timestamp_millis() as f64, &event_line)], + Some(RETRY_MAMAGER_PREFIX), + None, + ) + .await?; + + let event_serialized = serde_json::to_string(self)?; + + Self::put_index_hash_map( + Some(RETRY_MAMAGER_PREFIX), + &RETRY_MANAGER_STATE_INDEX, + &event_line, + event_serialized, + ) + .await?; + Ok(()) + } + + pub async fn check_uri(event_line: &str) -> Result, DynError> { + if let Some(post_details) = Self::check_sorted_set_member( + Some(RETRY_MAMAGER_PREFIX), + &RETRY_MANAGER_EVENTS_INDEX, + &[event_line], + ) + .await? + { + return Ok(Some(post_details)); + } + Ok(None) + } + + pub async fn get_from_index(pubky_uri: &str) -> Result, DynError> { + let mut found_event = None; + if let Some(event_state) = Self::get_index_hash_map( + Some(RETRY_MAMAGER_PREFIX), + &RETRY_MANAGER_STATE_INDEX, + pubky_uri, + ) + .await? + { + let event = serde_json::from_str::(&event_state)?; + found_event = Some(event); + } + Ok(found_event) + } +} diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs new file mode 100644 index 00000000..a45f5509 --- /dev/null +++ b/src/events/retry/manager.rs @@ -0,0 +1,91 @@ +use log::info; +use std::sync::Arc; +use tokio::sync::{ + mpsc::{Receiver, Sender}, + Mutex, +}; + +use crate::types::DynError; + +use super::event::RetryEvent; + +pub const CHANNEL_BUFFER: usize = 1024; + +#[derive(Debug, Clone)] +pub enum SenderMessage { + // Command to retry all pending events that are waiting to be indexed + Retry(String), + // Command to add a failed event to the RetryManager cache for future processing + Add(String, RetryEvent), +} + +pub type SenderChannel = Arc>>; +type ReceiverChannel = Arc>>; + +pub struct RetryManager { + pub sender: SenderChannel, + receiver: ReceiverChannel, +} + +/// Initializes a new `RetryManager` with a message-passing channel. +/// +/// This function sets up the `RetryManager` by taking a tuple containing +/// a `Sender` and `Receiver` from a multi-producer, single-consumer (mpsc) channel. +/// +/// # Parameters +/// - `(tx, rx)`: A tuple containing: +/// - `tx` (`Sender`): The sending half of the mpsc channel, used to dispatch messages to the manager. +/// - `rx` (`Receiver`): The receiving half of the mpsc channel, used to listen for incoming messages. +impl RetryManager { + pub fn initialise((tx, rx): (Sender, Receiver)) -> Self { + Self { + receiver: Arc::new(Mutex::new(rx)), + sender: Arc::new(Mutex::new(tx)), + } + } + + /// Executes the main event loop to process messages from the channel + /// This function listens for incoming messages on the receiver channel and handles them + /// based on their type: + /// - **`SenderMessage::Retry`**: Triggers the retry process + /// - **`SenderMessage::Add`**: Queues a failed event in the retry cache for future processing + /// + /// The loop runs continuously until the channel is closed, ensuring that all messages + /// are processed appropriately. + pub async fn exec(&self) -> Result<(), DynError> { + let mut rx = self.receiver.lock().await; + // Listen all the messages in the channel + while let Some(message) = rx.recv().await { + match message { + SenderMessage::Retry(homeserver_pubky) => { + self.retry_events_for_homeserver(&homeserver_pubky); + } + SenderMessage::Add(index_key, retry_event) => { + self.queue_failed_event(index_key, retry_event).await?; + } + } + } + Ok(()) + } + + fn retry_events_for_homeserver(&self, _homeserver_pubky: &str) { + // WIP: Retrieve the homeserver events from the SORTED SET + // RetryManager:events + } + + /// Stores the event line in the Redis cache, adding it to the retry queue for + /// future processing + /// # Arguments + /// - `index_key`: A `String` representing the compacted key for the event to be stored in Redis + /// - `retry_event`: A `RetryEvent` instance containing the details of the failed event + async fn queue_failed_event( + &self, + index_key: String, + retry_event: RetryEvent, + ) -> Result<(), DynError> { + info!("Add fail event to Redis: {}", index_key); + // Write in the index + retry_event.put_to_index(index_key).await?; + Ok(()) + } +} diff --git a/src/events/retry/mod.rs b/src/events/retry/mod.rs new file mode 100644 index 00000000..7003034b --- /dev/null +++ b/src/events/retry/mod.rs @@ -0,0 +1,2 @@ +pub mod event; +pub mod manager; diff --git a/src/watcher.rs b/src/watcher.rs index 1f25466d..aef77354 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -1,8 +1,7 @@ use log::error; use log::info; -use pubky_nexus::events::retry::RetryManager; -use pubky_nexus::events::retry::SenderMessage; -use pubky_nexus::events::retry::CHANNEL_BUFFER; +use pubky_nexus::events::retry::manager::SenderMessage; +use pubky_nexus::events::retry::manager::{RetryManager, CHANNEL_BUFFER}; use pubky_nexus::PubkyConnector; use pubky_nexus::{setup, Config, EventProcessor}; use tokio::sync::mpsc; diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 20635b67..baf6a8cf 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -5,7 +5,7 @@ use anyhow::Result; use chrono::Utc; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; -use pubky_nexus::events::{processor::EventErrorType, retry::RetryEvent, EventType}; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; // These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` // cache correctly adds the events as expected. @@ -37,9 +37,14 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { // => Create user tag let label = "peak_limit"; + let dependency_uri = format!( + "pubky://{author_id}/pub/pubky.app/posts/{}", + "0032Q4SFBFD4G" + ); + // Create a tag in a fake post let tag = PubkyAppTag { - uri: format!("pubky://{author_id}/pub/pubky.app/posts/{}", "0032Q4SFBFDG"), + uri: dependency_uri.clone(), label: label.to_string(), created_at: Utc::now().timestamp_millis(), }; @@ -56,35 +61,32 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { test.put(tag_url.as_str(), tag_blob.clone()).await?; tokio::time::sleep(Duration::from_millis(500)).await; - let homeserver_pubky = test.get_homeserver_pubky(); + let index_key = format!( + "{} {}", + EventType::Put, + RetryEvent::generate_index_key(&tag_url).unwrap() + ); // Assert if the event is in the timeline - let timeline = RetryEvent::check_uri(&homeserver_pubky, &tag_url) - .await - .unwrap(); - assert!(timeline.is_some()); + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); // Assert if the event is in the state hash map - let event_retry = RetryEvent::get_from_hash_map_index(&homeserver_pubky, &tag_url) - .await - .unwrap(); + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); let event_state = event_retry.unwrap(); - assert_eq!(event_state.uri, tag_url); - assert_eq!(event_state.event_type, EventType::Put); - assert_eq!(event_state.error_type, EventErrorType::MissingDependency); assert_eq!(event_state.retry_count, 0); - let dependency = format!( - "pubky://{author_id}/pub/pubky.app/posts/{}", - "0032Q4SFBFD4G" - ); - assert!(event_state.dependency.is_some()); - let dependency_list = event_state.dependency.unwrap(); - assert_eq!(dependency_list.len(), 1); - assert_eq!(dependency_list[0], dependency); + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], dependency_uri); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; Ok(()) } diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index b6192b05..72fc8186 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -5,7 +5,7 @@ use anyhow::Result; use chrono::Utc; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; -use pubky_nexus::events::{processor::EventErrorType, retry::RetryEvent, EventType}; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; // These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` // cache correctly adds the events as expected. @@ -50,32 +50,32 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { test.put(tag_url.as_str(), tag_blob.clone()).await?; tokio::time::sleep(Duration::from_millis(500)).await; - let homeserver_pubky = test.get_homeserver_pubky(); + let index_key = format!( + "{} {}", + EventType::Put, + RetryEvent::generate_index_key(&tag_url).unwrap() + ); // Assert if the event is in the timeline - let timeline = RetryEvent::check_uri(&homeserver_pubky, &tag_url) - .await - .unwrap(); - assert!(timeline.is_some()); + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); // Assert if the event is in the state hash map - let event_retry = RetryEvent::get_from_hash_map_index(&homeserver_pubky, &tag_url) - .await - .unwrap(); + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); let event_state = event_retry.unwrap(); - assert_eq!(event_state.uri, tag_url); - assert_eq!(event_state.event_type, EventType::Put); - assert_eq!(event_state.error_type, EventErrorType::MissingDependency); assert_eq!(event_state.retry_count, 0); - let dependency = format!("pubky://{shadow_user_id}/pub/pubky.app/profile.json"); - assert!(event_state.dependency.is_some()); - let dependency_list = event_state.dependency.unwrap(); - assert_eq!(dependency_list.len(), 1); - assert_eq!(dependency_list[0], dependency); + let dependency_uri = format!("pubky://{shadow_user_id}/pub/pubky.app/profile.json"); + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], dependency_uri); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; Ok(()) } diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index 54784ee9..bb07699a 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -7,10 +7,8 @@ use pubky_app_specs::{ }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; -use pubky_nexus::events::{ - retry::{RetryManager, CHANNEL_BUFFER}, - Event, -}; +use pubky_nexus::events::retry::manager::{RetryManager, CHANNEL_BUFFER}; +use pubky_nexus::events::Event; use pubky_nexus::types::{DynError, PubkyId}; use pubky_nexus::{setup, Config, EventProcessor, PubkyConnector}; use serde_json::to_vec; @@ -95,7 +93,7 @@ impl WatcherTest { Ok(()) } - pub fn get_homeserver_pubky(&self) -> PubkyId { + pub fn _get_homeserver_pubky(&self) -> PubkyId { let homeserver_pubky = self.homeserver.public_key().to_uri_string(); // Slice after the 3rd character let pubky_part = &homeserver_pubky["pk:".len()..]; From f2dba561de6049f675a63458674cb799df694a2c Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 14 Jan 2025 07:39:32 +0100 Subject: [PATCH 10/42] replace the HashMap structure with a JSON-based implementation --- benches/watcher.rs | 2 +- src/db/kv/index/hash_map.rs | 50 -------------------------- src/db/kv/index/mod.rs | 1 - src/db/kv/traits.rs | 38 -------------------- src/events/processor.rs | 2 +- src/events/retry/event.rs | 54 +++++++++++++++------------- tests/watcher/tags/retry_post_tag.rs | 2 +- tests/watcher/tags/retry_user_tag.rs | 2 +- 8 files changed, 34 insertions(+), 117 deletions(-) delete mode 100644 src/db/kv/index/hash_map.rs diff --git a/benches/watcher.rs b/benches/watcher.rs index 160ba2fa..a3eb3ca1 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -5,7 +5,7 @@ use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::{ - events::manager::{RetryManager, SenderChannel}, + events::retry::manager::{RetryManager, SenderChannel}, types::PubkyId, EventProcessor, }; diff --git a/src/db/kv/index/hash_map.rs b/src/db/kv/index/hash_map.rs deleted file mode 100644 index 0035d0df..00000000 --- a/src/db/kv/index/hash_map.rs +++ /dev/null @@ -1,50 +0,0 @@ -use crate::db::connectors::redis::get_redis_conn; -use crate::types::DynError; -use redis::AsyncCommands; - -/// Inserts a value into a Redis hash map under the specified prefix, key, and field -/// * `prefix` - A string slice representing the prefix for the key. This is typically used to group related keys in Redis -/// * `key` - A string slice representing the main key in the hash map -/// * `field` - A string slice representing the field within the hash map where the value will be stored -/// * `value` - A `String` containing the value to be stored in the hash map -pub async fn put(prefix: &str, key: &str, field: &str, value: String) -> Result<(), DynError> { - let index_key = format!("{}:{}", prefix, key); - let mut redis_conn = get_redis_conn().await?; - // HSETNX sets the field only if it does not exist. Returns 1 (true) or 0 (false). - let _: bool = redis_conn.hset(&index_key, field, value).await?; - Ok(()) -} - -/// Retrieves a value from a Redis hash map using the specified prefix, key, and field -/// # Arguments -/// * `prefix` - A string slice representing the prefix for the key. This is used to group related keys in Redis -/// * `key` - A string slice representing the main key in the hash map -/// * `field` - A string slice representing the field within the hash map from which the value will be retrieved -pub async fn get(prefix: &str, key: &str, field: &str) -> Result, DynError> { - let index_key = format!("{}:{}", prefix, key); - let mut redis_conn = get_redis_conn().await?; - - // HGET retrieves the value for the given field. - let value: Option = redis_conn.hget(&index_key, field).await?; - Ok(value) -} - -/// Deletes one or more fields from a Redis hash map under the specified prefix and key. -/// # Arguments -/// * `prefix` - A string slice representing the prefix for the key. This is used to group related keys in Redis -/// * `key` - A string slice representing the main key in the hash map -/// * `fields` - A slice of string slices representing the fields to be removed from the hash map -pub async fn _del(prefix: &str, key: &str, fields: &[&str]) -> Result<(), DynError> { - if fields.is_empty() { - return Ok(()); - } - - let index_key = format!("{}:{}", prefix, key); - let mut redis_conn = get_redis_conn().await?; - - // The HDEL command is used to remove one or more fields from a hash. - // It returns the number of fields that were removed - let _removed_count: i32 = redis_conn.hdel(index_key, fields).await?; - - Ok(()) -} diff --git a/src/db/kv/index/mod.rs b/src/db/kv/index/mod.rs index c8474db5..480a2796 100644 --- a/src/db/kv/index/mod.rs +++ b/src/db/kv/index/mod.rs @@ -1,4 +1,3 @@ -pub mod hash_map; /// Module for redis Indexing operations split into modules by Redis types pub mod json; pub mod lists; diff --git a/src/db/kv/traits.rs b/src/db/kv/traits.rs index 73950503..c34f48b5 100644 --- a/src/db/kv/traits.rs +++ b/src/db/kv/traits.rs @@ -681,42 +681,4 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { let key = key_parts.join(":"); sorted_sets::get_lex_range("Sorted", &key, min, max, skip, limit).await } - - // ############################################################ - // ############ HASH MAP related functions #################### - // ############################################################ - - /// Inserts a value into a Redis hash map at the specified key and field. - /// # Arguments - /// * `prefix` - An optional string slice representing the prefix for the key - /// * `key_parts` - A slice of string slices representing parts of the key - /// * `field` - A string slice representing the field in the hash map where the value will be stored. - /// * `value` - A `String` containing the value to be stored in the hash map - async fn put_index_hash_map( - prefix: Option<&str>, - key_parts: &[&str], - field: &str, - value: String, - ) -> Result<(), DynError> { - let prefix = prefix.unwrap_or(SORTED_PREFIX); - let key = key_parts.join(":"); - // Store the elements in the Redis sorted set - hash_map::put(prefix, &key, field, value).await - } - - /// Retrieves a value from a Redis hash map at the specified key and field. - /// # Arguments - /// * `prefix` - An optional string slice representing the prefix for the key - /// * `key_parts` - A slice of string slices representing parts of the key - /// * `field` - A string slice representing the field in the hash map from which the value will be retrieved - async fn get_index_hash_map( - prefix: Option<&str>, - key_parts: &[&str], - field: &str, - ) -> Result, DynError> { - let prefix = prefix.unwrap_or(SORTED_PREFIX); - let key = key_parts.join(":"); - // Store the elements in the Redis sorted set - hash_map::get(prefix, &key, field).await - } } diff --git a/src/events/processor.rs b/src/events/processor.rs index d4438fec..1b7e8066 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -164,7 +164,7 @@ impl EventProcessor { // This block is unlikely to be reached, as it would typically fail during the validation process. return Ok(()); }; - let index_key = format!("{} {}", event.event_type, index); + let index_key = format!("{}:{}", event.event_type, index); let sender = self.sender.lock().await; match sender .send(SenderMessage::Add(index_key, retry_event)) diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index 47f9cf57..f95e41cb 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -1,3 +1,4 @@ +use async_trait::async_trait; use chrono::Utc; use serde::{Deserialize, Serialize}; @@ -6,6 +7,7 @@ use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; pub const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; +pub const HOMESERVER_PROTOCOL: &str = "pubky:"; pub const HOMESERVER_PUBLIC_REPOSITORY: &str = "pub"; pub const HOMESERVER_APP_REPOSITORY: &str = "pubky.app"; @@ -17,7 +19,12 @@ pub struct RetryEvent { pub error_type: EventProcessorError, } -impl RedisOps for RetryEvent {} +#[async_trait] +impl RedisOps for RetryEvent { + async fn prefix() -> String { + String::from(RETRY_MAMAGER_PREFIX) + } +} impl RetryEvent { pub fn new(error_type: EventProcessorError) -> Self { @@ -28,22 +35,27 @@ impl RetryEvent { } /// It processes a homeserver URI and extracts specific components to form a index key - /// in the format `"{pubkyId}:{repository_model}/{event_id}"` + /// in the format `"{pubkyId}:{repository_model}:{event_id}"` /// # Parameters /// - `event_uri`: A string slice representing the event URI to be processed pub fn generate_index_key(event_uri: &str) -> Option { let parts: Vec<&str> = event_uri.split('/').collect(); if parts.len() >= 7 - && parts[0] == "pubky:" + && parts[0] == HOMESERVER_PROTOCOL && parts[3] == HOMESERVER_PUBLIC_REPOSITORY && parts[4] == HOMESERVER_APP_REPOSITORY { - Some(format!("{}:{}/{}", parts[2], parts[5], parts[6])) + Some(format!("{}:{}:{}", parts[2], parts[5], parts[6])) } else { None } } + /// Stores an event in both a sorted set and a JSON index in Redis. + /// It adds an event line to a Redis sorted set with a timestamp-based score + /// and also stores the event details in a separate JSON index for retrieval. + /// # Arguments + /// * `event_line` - A `String` representing the event line to be indexed. pub async fn put_to_index(&self, event_line: String) -> Result<(), DynError> { Self::put_index_sorted_set( &RETRY_MANAGER_EVENTS_INDEX, @@ -54,23 +66,20 @@ impl RetryEvent { ) .await?; - let event_serialized = serde_json::to_string(self)?; + let index = &[RETRY_MANAGER_STATE_INDEX, [&event_line]].concat(); + self.put_index_json(index, None).await?; - Self::put_index_hash_map( - Some(RETRY_MAMAGER_PREFIX), - &RETRY_MANAGER_STATE_INDEX, - &event_line, - event_serialized, - ) - .await?; Ok(()) } - pub async fn check_uri(event_line: &str) -> Result, DynError> { + /// Checks if a specific event exists in the Redis sorted set + /// # Arguments + /// * `event_index` - A `&str` representing the event index to check + pub async fn check_uri(event_index: &str) -> Result, DynError> { if let Some(post_details) = Self::check_sorted_set_member( Some(RETRY_MAMAGER_PREFIX), &RETRY_MANAGER_EVENTS_INDEX, - &[event_line], + &[event_index], ) .await? { @@ -79,17 +88,14 @@ impl RetryEvent { Ok(None) } - pub async fn get_from_index(pubky_uri: &str) -> Result, DynError> { + /// Retrieves an event from the JSON index in Redis based on its index + /// # Arguments + /// * `event_index` - A `&str` representing the event index to retrieve + pub async fn get_from_index(event_index: &str) -> Result, DynError> { let mut found_event = None; - if let Some(event_state) = Self::get_index_hash_map( - Some(RETRY_MAMAGER_PREFIX), - &RETRY_MANAGER_STATE_INDEX, - pubky_uri, - ) - .await? - { - let event = serde_json::from_str::(&event_state)?; - found_event = Some(event); + let index = &[RETRY_MANAGER_STATE_INDEX, [event_index]].concat(); + if let Some(fail_event) = Self::try_from_index_json(index).await? { + found_event = Some(fail_event); } Ok(found_event) } diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index baf6a8cf..7c5204de 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -62,7 +62,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( - "{} {}", + "{}:{}", EventType::Put, RetryEvent::generate_index_key(&tag_url).unwrap() ); diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 72fc8186..01994659 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -51,7 +51,7 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( - "{} {}", + "{}:{}", EventType::Put, RetryEvent::generate_index_key(&tag_url).unwrap() ); From 9abbe4e2aa204aa9699011ab49f1e64958c2116c Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 14 Jan 2025 12:20:07 +0100 Subject: [PATCH 11/42] Change the dependencies format: Instead of a URI, add RetryManager index format --- src/events/handlers/bookmark.rs | 2 +- src/events/handlers/follow.rs | 2 +- src/events/handlers/mute.rs | 2 +- src/events/handlers/post.rs | 13 ++++++++++--- src/events/handlers/tag.rs | 8 ++++---- src/events/processor.rs | 2 +- src/events/retry/event.rs | 24 +++++++++++++++++------- tests/watcher/posts/fail_reply.rs | 2 ++ tests/watcher/posts/fail_repost.rs | 2 ++ tests/watcher/posts/fail_user.rs | 2 ++ 10 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 13fe4ed6..100d43dc 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -42,7 +42,7 @@ pub async fn sync_put( OperationOutcome::Updated => true, // TODO: Should return an error that should be processed by RetryManager OperationOutcome::Pending => { - let dependency = vec![format!("pubky://{author_id}/pub/pubky.app/posts/{post_id}")]; + let dependency = vec![format!("{author_id}:posts:{post_id}")]; return Err(EventProcessorError::MissingDependency { dependency }.into()); } }; diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index a93d3411..df406450 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -25,7 +25,7 @@ pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), // Do not duplicate the follow relationship OperationOutcome::Updated => return Ok(()), OperationOutcome::Pending => { - let dependency = vec![format!("pubky://{followee_id}/pub/pubky.app/profile.json")]; + let dependency = vec![format!("{followee_id}:user:profile.json")]; return Err(EventProcessorError::MissingDependency { dependency }.into()); } // The relationship did not exist, create all related indexes diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index 35563aa4..2fdff73b 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -21,7 +21,7 @@ pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), OperationOutcome::Pending => { - let dependency = vec![format!("pubky://{muted_id}/pub/pubky.app/profile.json")]; + let dependency = vec![format!("{muted_id}:user:profile.json")]; Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index 6e441932..e54d9f0c 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -1,6 +1,7 @@ use crate::db::graph::exec::{exec_single_row, execute_graph_operation, OperationOutcome}; use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; +use crate::events::retry::event::RetryEvent; use crate::events::uri::ParsedUri; use crate::models::notification::{Notification, PostChangedSource, PostChangedType}; use crate::models::post::{ @@ -45,13 +46,19 @@ pub async fn sync_put( OperationOutcome::Pending => { let mut dependency = Vec::new(); if let Some(replied_uri) = &post_relationships.replied { - dependency.push(replied_uri.clone()); + let reply_dependency = RetryEvent::generate_index_key(replied_uri) + // This block is unlikely to be reached, as it would typically fail during the validation process + .unwrap_or_else(|| replied_uri.clone()); + dependency.push(reply_dependency); } if let Some(reposted_uri) = &post_relationships.reposted { - dependency.push(reposted_uri.clone()); + let reply_dependency = RetryEvent::generate_index_key(reposted_uri) + // This block is unlikely to be reached, as it would typically fail during the validation process + .unwrap_or_else(|| reposted_uri.clone()); + dependency.push(reply_dependency); } if dependency.is_empty() { - dependency.push(format!("pubky://{author_id}/pub/pubky.app/profile.json")) + dependency.push(format!("{author_id}:user:profile.json")) } return Err(EventProcessorError::MissingDependency { dependency }.into()); } diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 728fd533..29ed132a 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -81,7 +81,8 @@ async fn put_sync_post( { OperationOutcome::Updated => Ok(()), OperationOutcome::Pending => { - let dependency = vec![format!("pubky://{author_id}/pub/pubky.app/posts/{post_id}")]; + // Ensure that dependencies follow the same format as the RetryManager keys + let dependency = vec![format!("{author_id}:posts:{post_id}")]; Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { @@ -159,9 +160,8 @@ async fn put_sync_user( { OperationOutcome::Updated => Ok(()), OperationOutcome::Pending => { - let dependency = vec![format!( - "pubky://{tagged_user_id}/pub/pubky.app/profile.json" - )]; + // Ensure that dependencies follow the same format as the RetryManager keys + let dependency = vec![format!("{tagged_user_id}:user:profile.json")]; Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/processor.rs b/src/events/processor.rs index 1b7e8066..ae67ac82 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -161,7 +161,7 @@ impl EventProcessor { let index = if let Some(retry_index) = RetryEvent::generate_index_key(&event.uri) { retry_index } else { - // This block is unlikely to be reached, as it would typically fail during the validation process. + // This block is unlikely to be reached, as it would typically fail during the validation process return Ok(()); }; let index_key = format!("{}:{}", event.event_type, index); diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index f95e41cb..63526a8f 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -40,14 +40,24 @@ impl RetryEvent { /// - `event_uri`: A string slice representing the event URI to be processed pub fn generate_index_key(event_uri: &str) -> Option { let parts: Vec<&str> = event_uri.split('/').collect(); - if parts.len() >= 7 - && parts[0] == HOMESERVER_PROTOCOL - && parts[3] == HOMESERVER_PUBLIC_REPOSITORY - && parts[4] == HOMESERVER_APP_REPOSITORY + // Ensure the URI structure matches the expected format + if parts.first() != Some(&HOMESERVER_PROTOCOL) + || parts.get(3) != Some(&HOMESERVER_PUBLIC_REPOSITORY) + || parts.get(4) != Some(&HOMESERVER_APP_REPOSITORY) { - Some(format!("{}:{}:{}", parts[2], parts[5], parts[6])) - } else { - None + return None; + } + + match parts.as_slice() { + // Regular PubkyApp URIs + [_, _, pubky_id, _, _, domain, event_id] => { + Some(format!("{}:{}:{}", pubky_id, domain, event_id)) + } + // PubkyApp user profile URI (profile.json) + [_, _, pubky_id, _, _, "profile.json"] => { + Some(format!("{}:user:profile.json", pubky_id)) + } + _ => None, } } diff --git a/tests/watcher/posts/fail_reply.rs b/tests/watcher/posts/fail_reply.rs index bd0c6429..350fa233 100644 --- a/tests/watcher/posts/fail_reply.rs +++ b/tests/watcher/posts/fail_reply.rs @@ -6,6 +6,8 @@ use pubky_common::crypto::Keypair; use pubky_nexus::types::DynError; /// The user profile is stored in the homeserver and synched in the graph, but the posts just exist in the homeserver +// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), +// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_reply_without_post_parent() -> Result<(), DynError> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/posts/fail_repost.rs b/tests/watcher/posts/fail_repost.rs index 8c6236a4..6a69b0f9 100644 --- a/tests/watcher/posts/fail_repost.rs +++ b/tests/watcher/posts/fail_repost.rs @@ -6,6 +6,8 @@ use pubky_common::crypto::Keypair; use pubky_nexus::types::DynError; /// The user profile is stored in the homeserver and synched in the graph, but the posts just exist in the homeserver +// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), +// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_repost_without_post_parent() -> Result<(), DynError> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/posts/fail_user.rs b/tests/watcher/posts/fail_user.rs index ceade2ec..ab54009b 100644 --- a/tests/watcher/posts/fail_user.rs +++ b/tests/watcher/posts/fail_user.rs @@ -4,6 +4,8 @@ use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind}; use pubky_common::crypto::Keypair; /// The user profile is stored in the homeserver. Missing the author to connect the post +// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), +// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_without_user() -> Result<()> { let mut test = WatcherTest::setup().await?; From 0efe196c12f603cd22cdcf765c239b47a6d8e8bc Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 14 Jan 2025 17:20:17 +0100 Subject: [PATCH 12/42] add more integration tests --- src/watcher.rs | 1 + tests/watcher/bookmarks/mod.rs | 1 + tests/watcher/bookmarks/retry_bookmark.rs | 77 ++++++++++++++++++++++ tests/watcher/follows/mod.rs | 1 + tests/watcher/follows/retry_follow.rs | 66 +++++++++++++++++++ tests/watcher/mutes/mod.rs | 1 + tests/watcher/mutes/retry_mute.rs | 66 +++++++++++++++++++ tests/watcher/posts/mod.rs | 4 ++ tests/watcher/posts/retry_all.rs | 78 +++++++++++++++++++++++ tests/watcher/posts/retry_post.rs | 64 +++++++++++++++++++ tests/watcher/posts/retry_reply.rs | 72 +++++++++++++++++++++ tests/watcher/posts/retry_repost.rs | 75 ++++++++++++++++++++++ tests/watcher/tags/retry_post_tag.rs | 2 +- tests/watcher/tags/retry_user_tag.rs | 7 +- 14 files changed, 511 insertions(+), 4 deletions(-) create mode 100644 tests/watcher/bookmarks/retry_bookmark.rs create mode 100644 tests/watcher/follows/retry_follow.rs create mode 100644 tests/watcher/mutes/retry_mute.rs create mode 100644 tests/watcher/posts/retry_all.rs create mode 100644 tests/watcher/posts/retry_post.rs create mode 100644 tests/watcher/posts/retry_reply.rs create mode 100644 tests/watcher/posts/retry_repost.rs diff --git a/src/watcher.rs b/src/watcher.rs index aef77354..9f803488 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -24,6 +24,7 @@ async fn main() -> Result<(), Box> { // Prepare the sender channel to send the messages to the retry manager let sender_clone = retry_manager.sender.clone(); // Create new asynchronous task to control the failed events + // TODO: Assure the thread safety tokio::spawn(async move { let _ = retry_manager.exec().await; }); diff --git a/tests/watcher/bookmarks/mod.rs b/tests/watcher/bookmarks/mod.rs index 46cec66d..ac4965df 100644 --- a/tests/watcher/bookmarks/mod.rs +++ b/tests/watcher/bookmarks/mod.rs @@ -3,3 +3,4 @@ mod fail_index; mod raw; mod utils; mod viewer; +mod retry_bookmark; \ No newline at end of file diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs new file mode 100644 index 00000000..4dee635f --- /dev/null +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -0,0 +1,77 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::{traits::HashId, PubkyAppBookmark, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the post to connect the bookmark +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_bookmark_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let keypair = Keypair::random(); + let user = PubkyAppUser { + bio: Some("test_homeserver_bookmark_cannot_index".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:Bookmark:User".to_string(), + status: None, + }; + let user_id = test.create_user(&keypair, &user).await?; + + // Use a placeholder parent post ID to intentionally avoid resolving it in the graph database + let fake_post_id = "0032QB10HCRHG"; + let fake_user_id = "ba3e8qeby33uq9cughpxdf7bew9etn1eq8bc3yhwg7p1f54yaozy"; + // Create parent post uri + let post_uri = format!("pubky://{fake_user_id}/pub/pubky.app/posts/{fake_post_id}"); + + // Create a bookmark content + let bookmark = PubkyAppBookmark { + uri: post_uri, + created_at: chrono::Utc::now().timestamp_millis(), + }; + let bookmark_blob = serde_json::to_vec(&bookmark)?; + // Create the bookmark of the shadow user + let bookmark_id = bookmark.create_id(); + let bookmark_url = format!( + "pubky://{}/pub/pubky.app/bookmarks/{}", + user_id, bookmark_id + ); + // Put bookmark + test.put(&bookmark_url, bookmark_blob).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&bookmark_url).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + let dependency_uri = format!("{fake_user_id}:posts:{fake_post_id}"); + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], dependency_uri); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/follows/mod.rs b/tests/watcher/follows/mod.rs index 5c0f725a..fa871d9f 100644 --- a/tests/watcher/follows/mod.rs +++ b/tests/watcher/follows/mod.rs @@ -8,3 +8,4 @@ mod put_friends; mod put_notification; mod put_sequential; mod utils; +mod retry_follow; \ No newline at end of file diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs new file mode 100644 index 00000000..4a424894 --- /dev/null +++ b/tests/watcher/follows/retry_follow.rs @@ -0,0 +1,66 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::PubkyAppUser; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the followee to connect with follower +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_follow_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let followee_keypair = Keypair::random(); + let followee_id = followee_keypair.public_key().to_z32(); + // In that case, that user will act as a NotSyncUser or user not registered in pubky.app + // It will not have a profile.json + test.register_user(&followee_keypair).await?; + + let follower_keypair = Keypair::random(); + let follower_user = PubkyAppUser { + bio: Some("test_homeserver_follow_cannot_index".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:Follower".to_string(), + status: None, + }; + let follower_id = test.create_user(&follower_keypair, &follower_user).await?; + + // Mute the user + test.create_follow(&follower_id, &followee_id).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{follower_id}/pub/pubky.app/follows/{followee_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + let dependency_uri = format!("{followee_id}:user:profile.json"); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], dependency_uri) + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/mutes/mod.rs b/tests/watcher/mutes/mod.rs index 51dc8e21..b7bcfa80 100644 --- a/tests/watcher/mutes/mod.rs +++ b/tests/watcher/mutes/mod.rs @@ -1,4 +1,5 @@ mod del; mod fail_index; +mod retry_mute; mod put; mod utils; diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs new file mode 100644 index 00000000..eccd6be8 --- /dev/null +++ b/tests/watcher/mutes/retry_mute.rs @@ -0,0 +1,66 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::PubkyAppUser; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the mutee to connect with muter +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_mute_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let mutee_keypair = Keypair::random(); + let mutee_id = mutee_keypair.public_key().to_z32(); + // In that case, that user will act as a NotSyncUser or user not registered in pubky.app + // It will not have a profile.json + test.register_user(&mutee_keypair).await?; + + let muter_keypair = Keypair::random(); + let muter_user = PubkyAppUser { + bio: Some("test_homeserver_mute_cannot_index".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:Muter".to_string(), + status: None, + }; + let muter_id = test.create_user(&muter_keypair, &muter_user).await?; + + // Mute the user + test.create_mute(&muter_id, &mutee_id).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + let dependency_uri = format!("{mutee_id}:user:profile.json"); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], dependency_uri) + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/posts/mod.rs b/tests/watcher/posts/mod.rs index bfb554ac..948f3e74 100644 --- a/tests/watcher/posts/mod.rs +++ b/tests/watcher/posts/mod.rs @@ -24,4 +24,8 @@ mod reply_notification; mod reply_repost; mod repost; mod repost_notification; +mod retry_post; +mod retry_reply; +mod retry_repost; +mod retry_all; pub mod utils; diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs new file mode 100644 index 00000000..04a0e74b --- /dev/null +++ b/tests/watcher/posts/retry_all.rs @@ -0,0 +1,78 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the post to connect the new one +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let keypair = Keypair::random(); + + let user = PubkyAppUser { + bio: Some("test_homeserver_post_reply".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:PostRepost:User".to_string(), + status: None, + }; + + let user_id = test.create_user(&keypair, &user).await?; + + // Use a placeholder parent post ID to intentionally avoid resolving it in the graph database + let reply_fake_post_id = "0032QB10HCRHG"; + let repost_fake_post_id = "0032QB10HP6JJ"; + // Create parent post uri + let reply_uri = format!("pubky://{user_id}/pub/pubky.app/posts/{reply_fake_post_id}"); + let repost_uri = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_fake_post_id}"); + + let repost_reply_post = PubkyAppPost { + content: "Watcher:IndexFail:PostRepost:User:Reply".to_string(), + kind: PubkyAppPostKind::Short, + parent: Some(reply_uri.clone()), + embed: Some(PubkyAppPostEmbed { + kind: PubkyAppPostKind::Short, + uri: repost_uri.clone(), + }), + attachments: None, + }; + + let repost_reply_post_id = test.create_post(&user_id, &repost_reply_post).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 2); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&reply_uri).unwrap()); + assert_eq!(dependency[1], RetryEvent::generate_index_key(&repost_uri).unwrap()); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs new file mode 100644 index 00000000..12671dd4 --- /dev/null +++ b/tests/watcher/posts/retry_post.rs @@ -0,0 +1,64 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the author to connect the post +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_post_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let keypair = Keypair::random(); + let user_id = keypair.public_key().to_z32(); + + // In that case, that user will act as a NotSyncUser or user not registered in pubky.app + // It will not have a profile.json + test.register_user(&keypair).await?; + + let post = PubkyAppPost { + content: "Watcher:IndexFail:PostEvent:PostWithoutUser".to_string(), + kind: PubkyAppPostKind::Short, + parent: None, + embed: None, + attachments: None, + }; + + let post_id = test.create_post(&user_id, &post).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + let dependency_uri = format!("pubky://{user_id}/pub/pubky.app/profile.json"); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs new file mode 100644 index 00000000..16e9aa76 --- /dev/null +++ b/tests/watcher/posts/retry_reply.rs @@ -0,0 +1,72 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the post to connect the new one +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_post_reply_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let keypair = Keypair::random(); + + let user = PubkyAppUser { + bio: Some("test_homeserver_post_reply".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:PostReply:User".to_string(), + status: None, + }; + + let user_id = test.create_user(&keypair, &user).await?; + + // Use a placeholder parent post ID to intentionally avoid resolving it in the graph database + let parent_fake_post_id = "0032QB10HCRHG"; + // Create parent post uri + let dependency_uri = format!("pubky://{user_id}/pub/pubky.app/posts/{parent_fake_post_id}"); + + let reply_post = PubkyAppPost { + content: "Watcher:IndexFail:PostReply:User:Reply".to_string(), + kind: PubkyAppPostKind::Short, + parent: Some(dependency_uri.clone()), + embed: None, + attachments: None, + }; + + let reply_id = test.create_post(&user_id, &reply_post).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{reply_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs new file mode 100644 index 00000000..331ed350 --- /dev/null +++ b/tests/watcher/posts/retry_repost.rs @@ -0,0 +1,75 @@ +use std::time::Duration; + +use crate::watcher::utils::watcher::WatcherTest; +use anyhow::Result; +use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; +use pubky_common::crypto::Keypair; +use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; + +/// The user profile is stored in the homeserver. Missing the post to connect the new one +// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` +// cache correctly adds the events as expected. +#[tokio_shared_rt::test(shared)] +async fn test_homeserver_post_repost_cannot_index() -> Result<()> { + let mut test = WatcherTest::setup().await?; + + let keypair = Keypair::random(); + + let user = PubkyAppUser { + bio: Some("test_homeserver_post_reply".to_string()), + image: None, + links: None, + name: "Watcher:IndexFail:PostRepost:User".to_string(), + status: None, + }; + + let user_id = test.create_user(&keypair, &user).await?; + + // Use a placeholder parent post ID to intentionally avoid resolving it in the graph database + let repost_fake_post_id = "0032QB10HCRHG"; + // Create parent post uri + let dependency_uri = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_fake_post_id}"); + + let repost_post = PubkyAppPost { + content: "Watcher:IndexFail:PostRepost:User:Reply".to_string(), + kind: PubkyAppPostKind::Short, + parent: None, + embed: Some(PubkyAppPostEmbed { + kind: PubkyAppPostKind::Short, + uri: dependency_uri.clone(), + }), + attachments: None, + }; + + let repost_id = test.create_post(&user_id, &repost_post).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let index_key = format!( + "{}:{}", + EventType::Put, + RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{repost_id}")).unwrap() + ); + + // Assert if the event is in the timeline + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state hash map + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::MissingDependency { dependency } => { + assert_eq!(dependency.len(), 1); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + } + _ => assert!(false, "The error type has to be MissingDependency type"), + }; + + + Ok(()) +} diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 7c5204de..54c5ab1e 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -83,7 +83,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_uri); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); } _ => assert!(false, "The error type has to be MissingDependency type"), }; diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 01994659..a9688408 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -31,8 +31,10 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { // => Create user tag let label = "friendly"; + let dependency_uri = format!("pubky://{shadow_user_id}/pub/pubky.app/profile.json"); + let tag = PubkyAppTag { - uri: format!("pubky://{}/pub/pubky.app/profile.json", shadow_user_id), + uri: dependency_uri.clone(), label: label.to_string(), created_at: Utc::now().timestamp_millis(), }; @@ -68,11 +70,10 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { assert_eq!(event_state.retry_count, 0); - let dependency_uri = format!("pubky://{shadow_user_id}/pub/pubky.app/profile.json"); match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_uri); + assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); } _ => assert!(false, "The error type has to be MissingDependency type"), }; From 4487e36c300e44296c80384dce587b7d729efbdf Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 15 Jan 2025 05:06:56 +0100 Subject: [PATCH 13/42] improve graph operation state enum --- src/db/graph/exec.rs | 6 +++--- src/events/handlers/bookmark.rs | 2 +- src/events/handlers/follow.rs | 4 ++-- src/events/handlers/mute.rs | 4 ++-- src/events/handlers/post.rs | 4 ++-- src/events/handlers/tag.rs | 4 ++-- src/events/handlers/user.rs | 2 +- src/events/processor.rs | 10 ---------- 8 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/db/graph/exec.rs b/src/db/graph/exec.rs index b7aa9274..8d14ef2e 100644 --- a/src/db/graph/exec.rs +++ b/src/db/graph/exec.rs @@ -13,7 +13,7 @@ pub enum OperationOutcome { CreatedOrDeleted, /// A required node/relationship was not found, indicating a missing dependency /// (often due to the node/relationship not yet being indexed or otherwise unavailable). - Pending, + MissingDependency, } /// Executes a graph query expected to return exactly one row containing a boolean column named @@ -22,7 +22,7 @@ pub enum OperationOutcome { /// - `true` => Returns [`OperationOutcome::Updated`] /// - `false` => Returns [`OperationOutcome::CreatedOrDeleted`] /// -/// If no rows are returned, this function returns [`OperationOutcome::Pending`], typically +/// If no rows are returned, this function returns [`OperationOutcome::MissingDependency`], typically /// indicating a missing dependency or an unmatched query condition. pub async fn execute_graph_operation(query: Query) -> Result { let mut result; @@ -38,7 +38,7 @@ pub async fn execute_graph_operation(query: Query) -> Result Ok(OperationOutcome::Updated), false => Ok(OperationOutcome::CreatedOrDeleted), }, - None => Ok(OperationOutcome::Pending), + None => Ok(OperationOutcome::MissingDependency), } } diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 100d43dc..dd51fa10 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -41,7 +41,7 @@ pub async fn sync_put( OperationOutcome::CreatedOrDeleted => false, OperationOutcome::Updated => true, // TODO: Should return an error that should be processed by RetryManager - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { let dependency = vec![format!("{author_id}:posts:{post_id}")]; return Err(EventProcessorError::MissingDependency { dependency }.into()); } diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index df406450..74d67ad0 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -24,7 +24,7 @@ pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), match Followers::put_to_graph(&follower_id, &followee_id).await? { // Do not duplicate the follow relationship OperationOutcome::Updated => return Ok(()), - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { let dependency = vec![format!("{followee_id}:user:profile.json")]; return Err(EventProcessorError::MissingDependency { dependency }.into()); } @@ -71,7 +71,7 @@ pub async fn sync_del(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), match Followers::del_from_graph(&follower_id, &followee_id).await? { // Both users exists but they do not have that relationship OperationOutcome::Updated => Ok(()), - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { Err("WATCHER: Missing some dependency to index the model".into()) } OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index 2fdff73b..a860f17f 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -20,7 +20,7 @@ pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro // (user_id)-[:MUTED]->(muted_id) match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { let dependency = vec![format!("{muted_id}:user:profile.json")]; Err(EventProcessorError::MissingDependency { dependency }.into()) } @@ -43,7 +43,7 @@ pub async fn sync_del(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro match Muted::del_from_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), // TODO: Should return an error that should be processed by RetryManager - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { Err("WATCHER: Missing some dependency to index the model".into()) } OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index e54d9f0c..9f22604a 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -43,7 +43,7 @@ pub async fn sync_put( let existed = match post_details.put_to_graph(&post_relationships).await? { OperationOutcome::CreatedOrDeleted => false, OperationOutcome::Updated => true, - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { let mut dependency = Vec::new(); if let Some(replied_uri) = &post_relationships.replied { let reply_dependency = RetryEvent::generate_index_key(replied_uri) @@ -292,7 +292,7 @@ pub async fn del(author_id: PubkyId, post_id: String) -> Result<(), DynError> { } // TODO: Should return an error that should be processed by RetryManager // WIP: Create a custom error type to pass enough info to the RetryManager - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { return Err("WATCHER: Missing some dependency to index the model".into()) } }; diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 29ed132a..ac36df3a 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -80,7 +80,7 @@ async fn put_sync_post( .await? { OperationOutcome::Updated => Ok(()), - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { // Ensure that dependencies follow the same format as the RetryManager keys let dependency = vec![format!("{author_id}:posts:{post_id}")]; Err(EventProcessorError::MissingDependency { dependency }.into()) @@ -159,7 +159,7 @@ async fn put_sync_user( .await? { OperationOutcome::Updated => Ok(()), - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { // Ensure that dependencies follow the same format as the RetryManager keys let dependency = vec![format!("{tagged_user_id}:user:profile.json")]; Err(EventProcessorError::MissingDependency { dependency }.into()) diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index 9cd0279f..d532a6a2 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -69,7 +69,7 @@ pub async fn del(user_id: PubkyId) -> Result<(), DynError> { } // Should return an error that could not be inserted in the RetryManager // TODO: WIP, it will be fixed in the comming PRs the error messages - OperationOutcome::Pending => { + OperationOutcome::MissingDependency => { return Err("WATCHER: Missing some dependency to index the model".into()) } } diff --git a/src/events/processor.rs b/src/events/processor.rs index ae67ac82..8559653e 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -7,8 +7,6 @@ use crate::types::PubkyId; use crate::{models::homeserver::Homeserver, Config}; use log::{debug, error, info}; use reqwest::Client; -use serde::Deserialize; -use serde::Serialize; pub struct EventProcessor { http_client: Client, @@ -17,14 +15,6 @@ pub struct EventProcessor { pub sender: SenderChannel, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum EventErrorType { - NotResolveHomeserver, - PubkyClientError, - MissingDependency, - GraphError, -} - impl EventProcessor { pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { let homeserver = Homeserver::from_config(config).await?; From 2a175a3286881e3388ea9f15a14e8e175ce7ba67 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 15 Jan 2025 14:22:03 +0100 Subject: [PATCH 14/42] retry queue with DEL events, testing with bookmarks --- src/db/kv/traits.rs | 6 ++-- src/events/error.rs | 4 ++- src/events/handlers/bookmark.rs | 17 ++++++------ src/events/processor.rs | 2 +- src/events/retry/event.rs | 34 +++++++++++++++++++++-- src/events/retry/manager.rs | 31 +++++++++++++++------ src/models/post/bookmark.rs | 1 + src/models/post/stream.rs | 14 ++++++---- src/models/tag/search.rs | 2 +- src/watcher.rs | 2 +- tests/watcher/bookmarks/mod.rs | 2 +- tests/watcher/bookmarks/retry_bookmark.rs | 17 ++++++++++-- tests/watcher/follows/mod.rs | 2 +- tests/watcher/follows/retry_follow.rs | 8 ++++-- tests/watcher/mutes/mod.rs | 2 +- tests/watcher/mutes/retry_mute.rs | 8 ++++-- tests/watcher/posts/mod.rs | 2 +- tests/watcher/posts/retry_all.rs | 20 +++++++++---- tests/watcher/posts/retry_post.rs | 11 +++++--- tests/watcher/posts/retry_reply.rs | 15 ++++++---- tests/watcher/posts/retry_repost.rs | 15 ++++++---- tests/watcher/tags/retry_post_tag.rs | 5 +++- tests/watcher/tags/retry_user_tag.rs | 5 +++- 23 files changed, 158 insertions(+), 67 deletions(-) diff --git a/src/db/kv/traits.rs b/src/db/kv/traits.rs index c34f48b5..190e02d6 100644 --- a/src/db/kv/traits.rs +++ b/src/db/kv/traits.rs @@ -594,6 +594,7 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { /// /// # Arguments /// + /// * `prefix` - An optional string representing the prefix for the Redis keys. If `Some(String)`, the prefix will be used /// * `key_parts` - A slice of string slices that represent the parts used to form the key under which the sorted set is stored. /// * `items` - A slice of string slices representing the elements to be removed from the sorted set. /// @@ -601,6 +602,7 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { /// /// Returns an error if the operation fails, such as if the Redis connection is unavailable. async fn remove_from_index_sorted_set( + prefix: Option<&str>, key_parts: &[&str], items: &[&str], ) -> Result<(), DynError> { @@ -608,11 +610,11 @@ pub trait RedisOps: Serialize + DeserializeOwned + Send + Sync { return Ok(()); } + let prefix = prefix.unwrap_or(SORTED_PREFIX); // Create the key by joining the key parts let key = key_parts.join(":"); - // Call the sorted_sets::del function to remove the items from the sorted set - sorted_sets::del("Sorted", &key, items).await + sorted_sets::del(prefix, &key, items).await } /// Retrieves a range of elements from a Redis sorted set using the provided key parts. diff --git a/src/events/error.rs b/src/events/error.rs index 4d5a9837..84a40413 100644 --- a/src/events/error.rs +++ b/src/events/error.rs @@ -5,8 +5,10 @@ use thiserror::Error; pub enum EventProcessorError { #[error("The user could not be indexed in nexus")] UserNotSync, - #[error("The event could not be indexed because some graph dependency is missing")] + #[error("The event could not be indexed due to missing graph dependenciesz")] MissingDependency { dependency: Vec }, + #[error("The event appear to be unindexed. Verify the event in the retry queue")] + SkipIndexing, #[error("The event does not exist anymore in the homeserver")] ContentNotFound { dependency: String }, #[error("PubkyClient could not reach/resolve the homeserver")] diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index dd51fa10..027ce532 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -69,15 +69,16 @@ pub async fn del(user_id: PubkyId, bookmark_id: String) -> Result<(), DynError> pub async fn sync_del(user_id: PubkyId, bookmark_id: String) -> Result<(), DynError> { // DELETE FROM GRAPH let deleted_bookmark_info = Bookmark::del_from_graph(&user_id, &bookmark_id).await?; + // Ensure the bookmark exists in the graph before proceeding + let (post_id, author_id) = match deleted_bookmark_info { + Some(info) => info, + None => return Err(EventProcessorError::SkipIndexing.into()), + }; - if let Some((post_id, author_id)) = deleted_bookmark_info { - // DELETE FROM INDEXes - Bookmark::del_from_index(&user_id, &post_id, &author_id).await?; - - // Update user counts with the new bookmark - // Skip updating counts if bookmark was not found in graph - UserCounts::update(&user_id, "bookmarks", JsonAction::Decrement(1)).await?; - } + // DELETE FROM INDEXes + Bookmark::del_from_index(&user_id, &post_id, &author_id).await?; + // Update user counts + UserCounts::update(&user_id, "bookmarks", JsonAction::Decrement(1)).await?; Ok(()) } diff --git a/src/events/processor.rs b/src/events/processor.rs index 8559653e..5d9c455d 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -157,7 +157,7 @@ impl EventProcessor { let index_key = format!("{}:{}", event.event_type, index); let sender = self.sender.lock().await; match sender - .send(SenderMessage::Add(index_key, retry_event)) + .send(SenderMessage::ProcessEvent(index_key, retry_event)) .await { Ok(_) => { diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index 63526a8f..cc213a69 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -2,7 +2,11 @@ use async_trait::async_trait; use chrono::Utc; use serde::{Deserialize, Serialize}; -use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; +use crate::{ + events::{error::EventProcessorError, EventType}, + types::DynError, + RedisOps, +}; pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; @@ -11,11 +15,14 @@ pub const HOMESERVER_PROTOCOL: &str = "pubky:"; pub const HOMESERVER_PUBLIC_REPOSITORY: &str = "pub"; pub const HOMESERVER_APP_REPOSITORY: &str = "pubky.app"; +/// Represents an event in the retry queue and it is used to manage events that have failed +/// to process and need to be retried #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RetryEvent { - // Number of retries attempted + /// Retry attempts made for this event pub retry_count: u32, - // This determines how the event should be processed during the retry process + /// The type of error that caused the event to fail + /// This determines how the event should be processed during the retry process pub error_type: EventProcessorError, } @@ -61,6 +68,10 @@ impl RetryEvent { } } + pub fn generate_opposite_event_line(event_line: &str) -> String { + event_line.replace(&EventType::Del.to_string(), &EventType::Put.to_string()) + } + /// Stores an event in both a sorted set and a JSON index in Redis. /// It adds an event line to a Redis sorted set with a timestamp-based score /// and also stores the event details in a separate JSON index for retrieval. @@ -109,4 +120,21 @@ impl RetryEvent { } Ok(found_event) } + + /// Deletes event-related indices from the retry queue in Redis + /// # Arguments + /// - `event_line` - A `String` representing the event line that needs to be unindexed from the retry queue + pub async fn delete(&self, event_line: String) -> Result<(), DynError> { + let put_event_line = Self::generate_opposite_event_line(&event_line); + Self::remove_from_index_sorted_set( + Some(RETRY_MAMAGER_PREFIX), + &RETRY_MANAGER_EVENTS_INDEX, + &[&put_event_line], + ) + .await?; + + let index = &[RETRY_MANAGER_STATE_INDEX, [&put_event_line]].concat(); + Self::remove_from_index_multiple_json(&[index]).await?; + Ok(()) + } } diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index a45f5509..a13743fd 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -5,18 +5,22 @@ use tokio::sync::{ Mutex, }; -use crate::types::DynError; +use crate::{events::error::EventProcessorError, types::DynError}; use super::event::RetryEvent; pub const CHANNEL_BUFFER: usize = 1024; +/// Represents commands sent to the retry manager for handling events in the retry queue #[derive(Debug, Clone)] pub enum SenderMessage { // Command to retry all pending events that are waiting to be indexed - Retry(String), - // Command to add a failed event to the RetryManager cache for future processing - Add(String, RetryEvent), + RetryEvent(String), + /// Command to process a specific event in the retry queue + /// This action can either add an event to the queue or use the event as context to remove it from the queue + /// - `String`: The index or identifier of the event being processed + /// - `RetryEvent`: The event to be added to or processed for removal from the retry queue + ProcessEvent(String, RetryEvent), } pub type SenderChannel = Arc>>; @@ -57,10 +61,10 @@ impl RetryManager { // Listen all the messages in the channel while let Some(message) = rx.recv().await { match message { - SenderMessage::Retry(homeserver_pubky) => { + SenderMessage::RetryEvent(homeserver_pubky) => { self.retry_events_for_homeserver(&homeserver_pubky); } - SenderMessage::Add(index_key, retry_event) => { + SenderMessage::ProcessEvent(index_key, retry_event) => { self.queue_failed_event(index_key, retry_event).await?; } } @@ -83,9 +87,18 @@ impl RetryManager { index_key: String, retry_event: RetryEvent, ) -> Result<(), DynError> { - info!("Add fail event to Redis: {}", index_key); - // Write in the index - retry_event.put_to_index(index_key).await?; + match &retry_event.error_type { + EventProcessorError::MissingDependency { .. } => { + info!("Add fail event to Redis: {}", index_key); + // Write in the index + retry_event.put_to_index(index_key).await?; + } + EventProcessorError::SkipIndexing => { + retry_event.delete(index_key).await?; + } + _ => (), + }; + Ok(()) } } diff --git a/src/models/post/bookmark.rs b/src/models/post/bookmark.rs index 1bbaa7a9..85950473 100644 --- a/src/models/post/bookmark.rs +++ b/src/models/post/bookmark.rs @@ -159,6 +159,7 @@ impl Bookmark { return Ok(Some((post_id, author_id))); } } + // No rows matched the query. The pattern did not yield any results Ok(None) } diff --git a/src/models/post/stream.rs b/src/models/post/stream.rs index d71e9994..5d965c7a 100644 --- a/src/models/post/stream.rs +++ b/src/models/post/stream.rs @@ -518,7 +518,8 @@ impl PostStream { post_id: &str, ) -> Result<(), DynError> { let element = format!("{}:{}", author_id, post_id); - Self::remove_from_index_sorted_set(&POST_TIMELINE_KEY_PARTS, &[element.as_str()]).await + Self::remove_from_index_sorted_set(None, &POST_TIMELINE_KEY_PARTS, &[element.as_str()]) + .await } /// Adds the post to a Redis sorted set using the `indexed_at` timestamp as the score. @@ -534,7 +535,7 @@ impl PostStream { post_id: &str, ) -> Result<(), DynError> { let key_parts = [&POST_PER_USER_KEY_PARTS[..], &[author_id]].concat(); - Self::remove_from_index_sorted_set(&key_parts, &[post_id]).await + Self::remove_from_index_sorted_set(None, &key_parts, &[post_id]).await } /// Adds the post response to a Redis sorted set using the `indexed_at` timestamp as the score. @@ -560,7 +561,7 @@ impl PostStream { ) -> Result<(), DynError> { let key_parts = [&POST_REPLIES_PER_POST_KEY_PARTS[..], parent_post_key_parts].concat(); let element = format!("{}:{}", author_id, reply_id); - Self::remove_from_index_sorted_set(&key_parts, &[element.as_str()]).await + Self::remove_from_index_sorted_set(None, &key_parts, &[element.as_str()]).await } /// Adds the post to a Redis sorted set of replies per author using the `indexed_at` timestamp as the score. @@ -580,7 +581,7 @@ impl PostStream { post_id: &str, ) -> Result<(), DynError> { let key_parts = [&POST_REPLIES_PER_USER_KEY_PARTS[..], &[author_id]].concat(); - Self::remove_from_index_sorted_set(&key_parts, &[post_id]).await + Self::remove_from_index_sorted_set(None, &key_parts, &[post_id]).await } /// Adds a bookmark to Redis sorted set using the `indexed_at` timestamp as the score. @@ -604,7 +605,7 @@ impl PostStream { ) -> Result<(), DynError> { let key_parts = [&BOOKMARKS_USER_KEY_PARTS[..], &[bookmarker_id]].concat(); let post_key = format!("{}:{}", author_id, post_id); - Self::remove_from_index_sorted_set(&key_parts, &[&post_key]).await + Self::remove_from_index_sorted_set(None, &key_parts, &[&post_key]).await } /// Adds the post to a Redis sorted set using the total engagement as the score. @@ -631,7 +632,8 @@ impl PostStream { post_id: &str, ) -> Result<(), DynError> { let post_key = format!("{}:{}", author_id, post_id); - Self::remove_from_index_sorted_set(&POST_TOTAL_ENGAGEMENT_KEY_PARTS, &[&post_key]).await + Self::remove_from_index_sorted_set(None, &POST_TOTAL_ENGAGEMENT_KEY_PARTS, &[&post_key]) + .await } pub async fn update_index_score( diff --git a/src/models/tag/search.rs b/src/models/tag/search.rs index c39b7e12..6d424bd5 100644 --- a/src/models/tag/search.rs +++ b/src/models/tag/search.rs @@ -157,7 +157,7 @@ impl TagSearch { if label_taggers.is_none() { let key_parts = [&TAG_GLOBAL_POST_TIMELINE[..], &[tag_label]].concat(); let post_key = format!("{}:{}", author_id, post_id); - Self::remove_from_index_sorted_set(&key_parts, &[&post_key]).await?; + Self::remove_from_index_sorted_set(None, &key_parts, &[&post_key]).await?; } Ok(()) } diff --git a/src/watcher.rs b/src/watcher.rs index 9f803488..eda83e23 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -48,7 +48,7 @@ async fn main() -> Result<(), Box> { if RETRY_THRESHOLD == retry_failed_events { let sender = event_processor.sender.lock().await; let _ = sender - .send(SenderMessage::Retry( + .send(SenderMessage::RetryEvent( event_processor.homeserver.id.to_string(), )) .await; diff --git a/tests/watcher/bookmarks/mod.rs b/tests/watcher/bookmarks/mod.rs index ac4965df..b77d9430 100644 --- a/tests/watcher/bookmarks/mod.rs +++ b/tests/watcher/bookmarks/mod.rs @@ -1,6 +1,6 @@ mod del; mod fail_index; mod raw; +mod retry_bookmark; mod utils; mod viewer; -mod retry_bookmark; \ No newline at end of file diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index 4dee635f..e89c08a0 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -12,7 +12,7 @@ use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, #[tokio_shared_rt::test(shared)] async fn test_homeserver_bookmark_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; - + let keypair = Keypair::random(); let user = PubkyAppUser { bio: Some("test_homeserver_bookmark_cannot_index".to_string()), @@ -41,10 +41,10 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { "pubky://{}/pub/pubky.app/bookmarks/{}", user_id, bookmark_id ); - // Put bookmark + // PUT bookmark test.put(&bookmark_url, bookmark_blob).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, @@ -72,6 +72,17 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + // DEL bookmark + test.del(&bookmark_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + assert!(timestamp.is_none()); + + // Assert that the event does not exist in the JSON index. In that case PUT event + let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + assert!(event_retry.is_none()); Ok(()) } diff --git a/tests/watcher/follows/mod.rs b/tests/watcher/follows/mod.rs index fa871d9f..70ab8c20 100644 --- a/tests/watcher/follows/mod.rs +++ b/tests/watcher/follows/mod.rs @@ -7,5 +7,5 @@ mod put; mod put_friends; mod put_notification; mod put_sequential; +mod retry_follow; mod utils; -mod retry_follow; \ No newline at end of file diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 4a424894..8433878f 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -32,11 +32,14 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { // Mute the user test.create_follow(&follower_id, &followee_id).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{follower_id}/pub/pubky.app/follows/{followee_id}")).unwrap() + RetryEvent::generate_index_key(&format!( + "pubky://{follower_id}/pub/pubky.app/follows/{followee_id}" + )) + .unwrap() ); // Assert if the event is in the timeline @@ -61,6 +64,5 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/mutes/mod.rs b/tests/watcher/mutes/mod.rs index b7bcfa80..7d57bbf5 100644 --- a/tests/watcher/mutes/mod.rs +++ b/tests/watcher/mutes/mod.rs @@ -1,5 +1,5 @@ mod del; mod fail_index; -mod retry_mute; mod put; +mod retry_mute; mod utils; diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index eccd6be8..8af38c4b 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -32,11 +32,14 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { // Mute the user test.create_mute(&muter_id, &mutee_id).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}")).unwrap() + RetryEvent::generate_index_key(&format!( + "pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}" + )) + .unwrap() ); // Assert if the event is in the timeline @@ -61,6 +64,5 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/posts/mod.rs b/tests/watcher/posts/mod.rs index 948f3e74..14d85ff1 100644 --- a/tests/watcher/posts/mod.rs +++ b/tests/watcher/posts/mod.rs @@ -24,8 +24,8 @@ mod reply_notification; mod reply_repost; mod repost; mod repost_notification; +mod retry_all; mod retry_post; mod retry_reply; mod retry_repost; -mod retry_all; pub mod utils; diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index 04a0e74b..ef0fc397 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -12,7 +12,7 @@ use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; - + let keypair = Keypair::random(); let user = PubkyAppUser { @@ -45,11 +45,14 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { let repost_reply_post_id = test.create_post(&user_id, &repost_reply_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}")).unwrap() + RetryEvent::generate_index_key(&format!( + "pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}" + )) + .unwrap() ); // Assert if the event is in the timeline @@ -67,12 +70,17 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 2); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&reply_uri).unwrap()); - assert_eq!(dependency[1], RetryEvent::generate_index_key(&repost_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&reply_uri).unwrap() + ); + assert_eq!( + dependency[1], + RetryEvent::generate_index_key(&repost_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index 12671dd4..e49d966c 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -30,11 +30,12 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { let post_id = test.create_post(&user_id, &post).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}")).unwrap() + RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}")) + .unwrap() ); // Assert if the event is in the timeline @@ -54,11 +55,13 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&dependency_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 16e9aa76..8d9be19c 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -12,7 +12,7 @@ use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_reply_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; - + let keypair = Keypair::random(); let user = PubkyAppUser { @@ -40,11 +40,14 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { let reply_id = test.create_post(&user_id, &reply_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{reply_id}")).unwrap() + RetryEvent::generate_index_key(&format!( + "pubky://{user_id}/pub/pubky.app/posts/{reply_id}" + )) + .unwrap() ); // Assert if the event is in the timeline @@ -62,11 +65,13 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&dependency_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index 331ed350..a2874e02 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -12,7 +12,7 @@ use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_repost_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; - + let keypair = Keypair::random(); let user = PubkyAppUser { @@ -43,11 +43,14 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { let repost_id = test.create_post(&user_id, &repost_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; - + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{repost_id}")).unwrap() + RetryEvent::generate_index_key(&format!( + "pubky://{user_id}/pub/pubky.app/posts/{repost_id}" + )) + .unwrap() ); // Assert if the event is in the timeline @@ -65,11 +68,13 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&dependency_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; - Ok(()) } diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 54c5ab1e..18bb6356 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -83,7 +83,10 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&dependency_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index a9688408..7fdf1aa8 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -73,7 +73,10 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], RetryEvent::generate_index_key(&dependency_uri).unwrap()); + assert_eq!( + dependency[0], + RetryEvent::generate_index_key(&dependency_uri).unwrap() + ); } _ => assert!(false, "The error type has to be MissingDependency type"), }; From 155d7e3fccba7d0e52e849dd319f04256638569a Mon Sep 17 00:00:00 2001 From: tipogi Date: Mon, 20 Jan 2025 15:21:00 +0100 Subject: [PATCH 15/42] Add error control for DEL events when the index fail --- src/events/handlers/follow.rs | 4 +-- src/events/handlers/mute.rs | 5 +--- src/events/handlers/post.rs | 6 +---- src/events/handlers/tag.rs | 2 +- src/events/handlers/user.rs | 6 +---- src/events/processor.rs | 2 +- src/events/retry/event.rs | 5 ++++ src/events/retry/manager.rs | 12 +++++++-- tests/watcher/bookmarks/retry_bookmark.rs | 33 ++++++++++++++++------- tests/watcher/follows/retry_follow.rs | 33 ++++++++++++++++++++--- tests/watcher/mutes/retry_mute.rs | 33 ++++++++++++++++++++--- tests/watcher/posts/retry_all.rs | 33 ++++++++++++++++++++--- tests/watcher/posts/retry_post.rs | 31 +++++++++++++++++++-- tests/watcher/posts/retry_reply.rs | 33 ++++++++++++++++++++--- tests/watcher/posts/retry_repost.rs | 33 ++++++++++++++++++++--- tests/watcher/tags/retry_post_tag.rs | 26 ++++++++++++++++++ tests/watcher/tags/retry_user_tag.rs | 26 ++++++++++++++++++ 17 files changed, 271 insertions(+), 52 deletions(-) diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index 74d67ad0..d897f994 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -71,9 +71,7 @@ pub async fn sync_del(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), match Followers::del_from_graph(&follower_id, &followee_id).await? { // Both users exists but they do not have that relationship OperationOutcome::Updated => Ok(()), - OperationOutcome::MissingDependency => { - Err("WATCHER: Missing some dependency to index the model".into()) - } + OperationOutcome::MissingDependency => Err(EventProcessorError::SkipIndexing.into()), OperationOutcome::CreatedOrDeleted => { // Check if the users are friends. Is this a break? :( let were_friends = Friends::check(&follower_id, &followee_id).await?; diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index a860f17f..12893a76 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -42,10 +42,7 @@ pub async fn sync_del(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro // DELETE FROM GRAPH match Muted::del_from_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), - // TODO: Should return an error that should be processed by RetryManager - OperationOutcome::MissingDependency => { - Err("WATCHER: Missing some dependency to index the model".into()) - } + OperationOutcome::MissingDependency => Err(EventProcessorError::SkipIndexing.into()), OperationOutcome::CreatedOrDeleted => { // REMOVE FROM INDEX Muted(vec![muted_id.to_string()]) diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index 9f22604a..8e68ddd5 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -290,11 +290,7 @@ pub async fn del(author_id: PubkyId, post_id: String) -> Result<(), DynError> { sync_put(dummy_deleted_post, author_id, post_id).await?; } - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager - OperationOutcome::MissingDependency => { - return Err("WATCHER: Missing some dependency to index the model".into()) - } + OperationOutcome::MissingDependency => return Err(EventProcessorError::SkipIndexing.into()), }; Ok(()) diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index ac36df3a..0b4d1e99 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -216,7 +216,7 @@ pub async fn del(user_id: PubkyId, tag_id: String) -> Result<(), DynError> { } else { // TODO: Should return an error that should be processed by RetryManager // WIP: Create a custom error type to pass enough info to the RetryManager - return Err("WATCHER: Missing some dependency to index the model".into()); + return Err(EventProcessorError::SkipIndexing.into()); } Ok(()) } diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index d532a6a2..b7ed647e 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -67,11 +67,7 @@ pub async fn del(user_id: PubkyId) -> Result<(), DynError> { sync_put(deleted_user, user_id).await?; } - // Should return an error that could not be inserted in the RetryManager - // TODO: WIP, it will be fixed in the comming PRs the error messages - OperationOutcome::MissingDependency => { - return Err("WATCHER: Missing some dependency to index the model".into()) - } + OperationOutcome::MissingDependency => return Err(EventProcessorError::SkipIndexing.into()), } // TODO notifications for deleted user diff --git a/src/events/processor.rs b/src/events/processor.rs index 5d9c455d..bf2e5cb8 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -161,7 +161,7 @@ impl EventProcessor { .await { Ok(_) => { - info!("Message send succesfully from the channel"); + info!("Message send to the RetryManager succesfully from the channel"); // TODO: Investigate non-blocking alternatives // The current use of `tokio::time::sleep` is intended to handle a situation where tasks in other threads // are not being tracked. This could potentially lead to issues with writing `RetryEvents` in certain cases. diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index cc213a69..d48b784e 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -68,6 +68,11 @@ impl RetryEvent { } } + /// Generates the opposite event line by replacing the event type, effectively + /// converting delete operations into put operations + /// + /// # Arguments + /// * `event_line` - A string slice representing the event line. pub fn generate_opposite_event_line(event_line: &str) -> String { event_line.replace(&EventType::Del.to_string(), &EventType::Put.to_string()) } diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index a13743fd..92814e41 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -89,12 +89,20 @@ impl RetryManager { ) -> Result<(), DynError> { match &retry_event.error_type { EventProcessorError::MissingDependency { .. } => { - info!("Add fail event to Redis: {}", index_key); + info!( + "Add PUT event to RetryManager, missing dependency for event: {}", + index_key + ); // Write in the index retry_event.put_to_index(index_key).await?; } EventProcessorError::SkipIndexing => { - retry_event.delete(index_key).await?; + info!( + "Add DEL event to RetryManager, it could not find the delete node: {}", + index_key + ); + // Write in the index + retry_event.put_to_index(index_key).await?; } _ => (), }; diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index e89c08a0..96126c00 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -45,18 +45,18 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { test.put(&bookmark_url, bookmark_blob).await?; tokio::time::sleep(Duration::from_millis(500)).await; - let index_key = format!( + let put_index_key = format!( "{}:{}", EventType::Put, RetryEvent::generate_index_key(&bookmark_url).unwrap() ); // Assert if the event is in the timeline - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); + let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map - let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&put_index_key).await.unwrap(); assert!(event_retry.is_some()); let event_state = event_retry.unwrap(); @@ -76,13 +76,28 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { test.del(&bookmark_url).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&bookmark_url).unwrap() + ); + // Assert that the event does not exist in the sorted set. In that case PUT event - let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); - assert!(timestamp.is_none()); + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); - // Assert that the event does not exist in the JSON index. In that case PUT event - let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); - assert!(event_retry.is_none()); + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; Ok(()) } diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 8433878f..7e00d619 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -33,13 +33,12 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { test.create_follow(&follower_id, &followee_id).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let follow_url = format!("pubky://{follower_id}/pub/pubky.app/follows/{followee_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!( - "pubky://{follower_id}/pub/pubky.app/follows/{followee_id}" - )) - .unwrap() + RetryEvent::generate_index_key(&follow_url).unwrap() ); // Assert if the event is in the timeline @@ -64,5 +63,31 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&follow_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&follow_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 8af38c4b..09f8e47f 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -33,13 +33,12 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { test.create_mute(&muter_id, &mutee_id).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let mute_url = format!("pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!( - "pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}" - )) - .unwrap() + RetryEvent::generate_index_key(&mute_url).unwrap() ); // Assert if the event is in the timeline @@ -64,5 +63,31 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&mute_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&mute_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index ef0fc397..767d80b2 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -46,13 +46,12 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { let repost_reply_post_id = test.create_post(&user_id, &repost_reply_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let repost_reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!( - "pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}" - )) - .unwrap() + RetryEvent::generate_index_key(&repost_reply_url).unwrap() ); // Assert if the event is in the timeline @@ -82,5 +81,31 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&repost_reply_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&repost_reply_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index e49d966c..caf708ec 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -31,11 +31,12 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { let post_id = test.create_post(&user_id, &post).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let post_url = format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}")) - .unwrap() + RetryEvent::generate_index_key(&post_url).unwrap() ); // Assert if the event is in the timeline @@ -63,5 +64,31 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&post_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&post_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 8d9be19c..1ffe1531 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -41,13 +41,12 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { let reply_id = test.create_post(&user_id, &reply_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{reply_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!( - "pubky://{user_id}/pub/pubky.app/posts/{reply_id}" - )) - .unwrap() + RetryEvent::generate_index_key(&reply_url).unwrap() ); // Assert if the event is in the timeline @@ -73,5 +72,31 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&reply_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&reply_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index a2874e02..aed853d4 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -44,13 +44,12 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { let repost_id = test.create_post(&user_id, &repost_post).await?; tokio::time::sleep(Duration::from_millis(500)).await; + let repost_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_id}"); + let index_key = format!( "{}:{}", EventType::Put, - RetryEvent::generate_index_key(&format!( - "pubky://{user_id}/pub/pubky.app/posts/{repost_id}" - )) - .unwrap() + RetryEvent::generate_index_key(&repost_url).unwrap() ); // Assert if the event is in the timeline @@ -76,5 +75,31 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&repost_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&repost_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 18bb6356..73091440 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -91,5 +91,31 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&tag_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&tag_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 7fdf1aa8..395682ff 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -81,5 +81,31 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { _ => assert!(false, "The error type has to be MissingDependency type"), }; + test.del(&tag_url).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + + let del_index_key = format!( + "{}:{}", + EventType::Del, + RetryEvent::generate_index_key(&tag_url).unwrap() + ); + + // Assert that the event does not exist in the sorted set. In that case PUT event + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); + assert!(timestamp.is_some()); + + // Assert if the event is in the state. JSON + let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); + assert!(event_retry.is_some()); + + let event_state = event_retry.unwrap(); + + assert_eq!(event_state.retry_count, 0); + + match event_state.error_type { + EventProcessorError::SkipIndexing => (), + _ => assert!(false, "The error type has to be SkipIndexing type"), + }; + Ok(()) } From e26935f58a538945ae347b38bc403d63edd36434 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 21 Jan 2025 18:34:56 +0100 Subject: [PATCH 16/42] log all errors related to event processing --- src/events/error.rs | 29 +++++++++++++++++------------ src/events/handlers/user.rs | 7 ++++++- src/events/mod.rs | 25 ++++++++++++++++++------- src/events/processor.rs | 28 ++++++++++++++++------------ src/events/retry/manager.rs | 26 ++++---------------------- 5 files changed, 61 insertions(+), 54 deletions(-) diff --git a/src/events/error.rs b/src/events/error.rs index 84a40413..8965d596 100644 --- a/src/events/error.rs +++ b/src/events/error.rs @@ -3,18 +3,23 @@ use thiserror::Error; #[derive(Error, Debug, Clone, Serialize, Deserialize)] pub enum EventProcessorError { - #[error("The user could not be indexed in nexus")] - UserNotSync, - #[error("The event could not be indexed due to missing graph dependenciesz")] + // Failed to execute query in the graph database + #[error("GraphQueryFailed: {message}")] + GraphQueryFailed { message: String }, + // The event could not be indexed due to missing graph dependencies + #[error("MissingDependency: Could not be indexed")] MissingDependency { dependency: Vec }, - #[error("The event appear to be unindexed. Verify the event in the retry queue")] + // The event appear to be unindexed. Verify the event in the retry queue + #[error("SkipIndexing: The PUT event appears to be unindexed")] SkipIndexing, - #[error("The event does not exist anymore in the homeserver")] - ContentNotFound { dependency: String }, - #[error("PubkyClient could not reach/resolve the homeserver")] - NotResolvedHomeserver, - #[error("The event could not be parsed from a line")] - InvalidEvent, - #[error("")] - PubkyClientError, + // The event could not be parsed from a line + #[error("InvalidEvent: {message}")] + InvalidEvent { message: String }, + // The Pubky client could not resolve the pubky + #[error("PubkyClientError: {message}")] + PubkyClientError { message: String }, + // #[error("The event does not exist anymore in the homeserver")] + // ContentNotFound { dependency: String }, + // #[error("PubkyClient could not reach/resolve the homeserver")] + // NotResolvedHomeserver, } diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index 5ee3d3f5..37d5e19a 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -27,7 +27,12 @@ pub async fn sync_put(user: PubkyAppUser, user_id: PubkyId) -> Result<(), DynErr // SAVE TO GRAPH match user_details.put_to_graph().await { Ok(_) => (), - Err(_) => return Err(EventProcessorError::UserNotSync.into()), + Err(e) => { + return Err(EventProcessorError::GraphQueryFailed { + message: format!("{:?}", e), + } + .into()) + } } // SAVE TO INDEX let user_id = user_details.id.clone(); diff --git a/src/events/mod.rs b/src/events/mod.rs index ac50d480..e1a40700 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -4,7 +4,8 @@ use crate::{ db::connectors::pubky::PubkyConnector, types::{DynError, PubkyId}, }; -use log::{debug, error}; +use error::EventProcessorError; +use log::debug; use reqwest; use serde::{Deserialize, Serialize}; use uri::ParsedUri; @@ -75,14 +76,20 @@ impl Event { debug!("New event: {}", line); let parts: Vec<&str> = line.split(' ').collect(); if parts.len() != 2 { - return Err(format!("Malformed event line: {}", line).into()); + return Err(EventProcessorError::InvalidEvent { + message: format!("Malformed event line: {}", line), + } + .into()); } let event_type = match parts[0] { "PUT" => EventType::Put, "DEL" => EventType::Del, _ => { - return Err(format!("Unknown event type: {}", parts[0]).into()); + return Err(EventProcessorError::InvalidEvent { + message: format!("Unknown event type: {}", parts[0]), + } + .into()) } }; @@ -123,8 +130,10 @@ impl Event { _ if uri.contains("/last_read") => return Ok(None), _ if uri.contains("/settings") => return Ok(None), _ => { - error!("Unrecognized resource in URI: {}", uri); - return Err("Unrecognized resource in URI".into()); + return Err(EventProcessorError::InvalidEvent { + message: format!("Unrecognized resource in URI: {}", uri), + } + .into()) } }; @@ -153,8 +162,10 @@ impl Event { let response = match pubky_client.get(url).send().await { Ok(response) => response, Err(e) => { - error!("WATCHER: Failed to fetch content at {}: {}", self.uri, e); - return Err(e.into()); + return Err(EventProcessorError::PubkyClientError { + message: format!("{}", e), + } + .into()) } }; diff --git a/src/events/processor.rs b/src/events/processor.rs index 00979c41..fac6cef7 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -115,7 +115,7 @@ impl EventProcessor { let event = match Event::parse_event(line) { Ok(event) => event, Err(e) => { - error!("Error while creating event line from line: {}", e); + error!("{}", e); None } }; @@ -134,22 +134,26 @@ impl EventProcessor { match event.clone().handle().await { Ok(_) => Ok(()), Err(e) => { - error!("Error while handling event: {}", e); - let retry_event = match e.downcast_ref::() { Some(event_processor_error) => RetryEvent::new(event_processor_error.clone()), - // Other retry errors must be ignored - _ => return Ok(()), + // Others errors must be logged at least for now + None => { + error!("Unhandled error type for URI: {}. {:?}", event.uri, e); + return Ok(()); + } }; // Generate a compress index to save in the cache - let index = if let Some(retry_index) = RetryEvent::generate_index_key(&event.uri) { - retry_index - } else { - // This block is unlikely to be reached, as it would typically fail during the validation process - return Ok(()); + let index = match RetryEvent::generate_index_key(&event.uri) { + Some(retry_index) => retry_index, + None => { + // Unlikely to be reached, as it would typically fail during the validation process + return Ok(()); + } }; let index_key = format!("{}:{}", event.event_type, index); + + // Send event to the retry manager let sender = self.sender.lock().await; match sender .send(SenderMessage::ProcessEvent(index_key, retry_event)) @@ -158,7 +162,7 @@ impl EventProcessor { Ok(_) => { info!("Message send to the RetryManager succesfully from the channel"); // TODO: Investigate non-blocking alternatives - // The current use of `tokio::time::sleep` is intended to handle a situation where tasks in other threads + // The current use of `tokio::time::sleep` (in the watcher tests) is intended to handle a situation where tasks in other threads // are not being tracked. This could potentially lead to issues with writing `RetryEvents` in certain cases. // Considerations: // - Determine if this delay is genuinely necessary. Testing may reveal that the `RetryManager` thread @@ -169,7 +173,7 @@ impl EventProcessor { // For now, the sleep is deactivated //tokio::time::sleep(Duration::from_millis(500)).await; } - Err(e) => error!("Err, {:?}", e), + Err(e) => error!("Failed to send message to RetryManager: {:?}", e), } Ok(()) } diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index 92814e41..c701e782 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -1,11 +1,11 @@ -use log::info; +use log::error; use std::sync::Arc; use tokio::sync::{ mpsc::{Receiver, Sender}, Mutex, }; -use crate::{events::error::EventProcessorError, types::DynError}; +use crate::types::DynError; use super::event::RetryEvent; @@ -87,26 +87,8 @@ impl RetryManager { index_key: String, retry_event: RetryEvent, ) -> Result<(), DynError> { - match &retry_event.error_type { - EventProcessorError::MissingDependency { .. } => { - info!( - "Add PUT event to RetryManager, missing dependency for event: {}", - index_key - ); - // Write in the index - retry_event.put_to_index(index_key).await?; - } - EventProcessorError::SkipIndexing => { - info!( - "Add DEL event to RetryManager, it could not find the delete node: {}", - index_key - ); - // Write in the index - retry_event.put_to_index(index_key).await?; - } - _ => (), - }; - + error!("{}: {}", retry_event.error_type, index_key); + retry_event.put_to_index(index_key).await?; Ok(()) } } From 02c9d608ae2550b18991f582af396f4f13b6f9a5 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 22 Jan 2025 12:42:11 +0100 Subject: [PATCH 17/42] Create a better error handling while we analyze the event line --- src/events/error.rs | 4 +- src/events/mod.rs | 51 +++++++++++++++++------- src/events/processor.rs | 7 +++- src/events/retry/manager.rs | 2 +- src/events/uri.rs | 78 ++++++++++++++++++++++++++----------- 5 files changed, 102 insertions(+), 40 deletions(-) diff --git a/src/events/error.rs b/src/events/error.rs index 8965d596..c9a6a107 100644 --- a/src/events/error.rs +++ b/src/events/error.rs @@ -13,8 +13,8 @@ pub enum EventProcessorError { #[error("SkipIndexing: The PUT event appears to be unindexed")] SkipIndexing, // The event could not be parsed from a line - #[error("InvalidEvent: {message}")] - InvalidEvent { message: String }, + #[error("InvalidEventLine: {message}")] + InvalidEventLine { message: String }, // The Pubky client could not resolve the pubky #[error("PubkyClientError: {message}")] PubkyClientError { message: String }, diff --git a/src/events/mod.rs b/src/events/mod.rs index e1a40700..21207fea 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -76,8 +76,8 @@ impl Event { debug!("New event: {}", line); let parts: Vec<&str> = line.split(' ').collect(); if parts.len() != 2 { - return Err(EventProcessorError::InvalidEvent { - message: format!("Malformed event line: {}", line), + return Err(EventProcessorError::InvalidEventLine { + message: format!("Malformed event line, {}", line), } .into()); } @@ -86,15 +86,16 @@ impl Event { "PUT" => EventType::Put, "DEL" => EventType::Del, _ => { - return Err(EventProcessorError::InvalidEvent { - message: format!("Unknown event type: {}", parts[0]), + return Err(EventProcessorError::InvalidEventLine { + message: format!("Unknown event type, {}", line), } .into()) } }; let uri = parts[1].to_string(); - let parsed_uri = ParsedUri::try_from(uri.as_str()).unwrap_or_default(); + let parsed_uri = ParsedUri::try_from(uri.as_str())?; + println!("ParsedURI: {:?}", parsed_uri); //TODO: This conversion to a match statement that only uses IF conditions is silly. // We could be patter matching the split test for "posts", "follows", etc maybe? @@ -104,34 +105,58 @@ impl Event { }, _ if uri.contains("/posts/") => ResourceType::Post { author_id: parsed_uri.user_id, - post_id: parsed_uri.post_id.ok_or("Missing post_id")?, + post_id: parsed_uri + .post_id + .ok_or(EventProcessorError::InvalidEventLine { + message: format!("Missing post_id, {} {}", parts[0], uri), + })?, }, _ if uri.contains("/follows/") => ResourceType::Follow { follower_id: parsed_uri.user_id, - followee_id: parsed_uri.follow_id.ok_or("Missing followee_id")?, + followee_id: parsed_uri + .follow_id + .ok_or(EventProcessorError::InvalidEventLine { + message: format!("Missing followee_id, {} {}", parts[0], uri), + })?, }, _ if uri.contains("/mutes/") => ResourceType::Mute { user_id: parsed_uri.user_id, - muted_id: parsed_uri.muted_id.ok_or("Missing muted_id")?, + muted_id: parsed_uri + .muted_id + .ok_or(EventProcessorError::InvalidEventLine { + message: format!("Missing muted_id, {} {}", parts[0], uri), + })?, }, _ if uri.contains("/bookmarks/") => ResourceType::Bookmark { user_id: parsed_uri.user_id, - bookmark_id: parsed_uri.bookmark_id.ok_or("Missing bookmark_id")?, + bookmark_id: parsed_uri.bookmark_id.ok_or( + EventProcessorError::InvalidEventLine { + message: format!("Missing bookmark_id, {} {}", parts[0], uri), + }, + )?, }, _ if uri.contains("/tags/") => ResourceType::Tag { user_id: parsed_uri.user_id, - tag_id: parsed_uri.tag_id.ok_or("Missing tag_id")?, + tag_id: parsed_uri + .tag_id + .ok_or(EventProcessorError::InvalidEventLine { + message: format!("Missing tag_id, {} {}", parts[0], uri), + })?, }, _ if uri.contains("/files/") => ResourceType::File { user_id: parsed_uri.user_id, - file_id: parsed_uri.file_id.ok_or("Missing file_id")?, + file_id: parsed_uri + .file_id + .ok_or(EventProcessorError::InvalidEventLine { + message: format!("Missing file_id, {} {}", parts[0], uri), + })?, }, _ if uri.contains("/blobs") => return Ok(None), _ if uri.contains("/last_read") => return Ok(None), _ if uri.contains("/settings") => return Ok(None), _ => { - return Err(EventProcessorError::InvalidEvent { - message: format!("Unrecognized resource in URI: {}", uri), + return Err(EventProcessorError::InvalidEventLine { + message: format!("Unrecognized resource in URI, {} {}", parts[0], uri), } .into()) } diff --git a/src/events/processor.rs b/src/events/processor.rs index fac6cef7..92a71fd3 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -135,10 +135,14 @@ impl EventProcessor { Ok(_) => Ok(()), Err(e) => { let retry_event = match e.downcast_ref::() { + Some(EventProcessorError::InvalidEventLine { message }) => { + error!("{}", message); + return Ok(()); + } Some(event_processor_error) => RetryEvent::new(event_processor_error.clone()), // Others errors must be logged at least for now None => { - error!("Unhandled error type for URI: {}. {:?}", event.uri, e); + error!("Unhandled error type for URI: {}, {:?}", event.uri, e); return Ok(()); } }; @@ -160,7 +164,6 @@ impl EventProcessor { .await { Ok(_) => { - info!("Message send to the RetryManager succesfully from the channel"); // TODO: Investigate non-blocking alternatives // The current use of `tokio::time::sleep` (in the watcher tests) is intended to handle a situation where tasks in other threads // are not being tracked. This could potentially lead to issues with writing `RetryEvents` in certain cases. diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index c701e782..9a5463a1 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -87,7 +87,7 @@ impl RetryManager { index_key: String, retry_event: RetryEvent, ) -> Result<(), DynError> { - error!("{}: {}", retry_event.error_type, index_key); + error!("{}, {}", retry_event.error_type, index_key); retry_event.put_to_index(index_key).await?; Ok(()) } diff --git a/src/events/uri.rs b/src/events/uri.rs index 51ad18fa..82623567 100644 --- a/src/events/uri.rs +++ b/src/events/uri.rs @@ -2,6 +2,8 @@ use crate::types::DynError; use crate::types::PubkyId; use std::convert::TryFrom; +use super::error::EventProcessorError; + #[derive(Default, Debug)] pub struct ParsedUri { pub user_id: PubkyId, @@ -21,54 +23,72 @@ impl TryFrom<&str> for ParsedUri { // Ensure the URI starts with the correct prefix if !uri.starts_with("pubky://") { - return Err("Invalid URI, must start with pubky://".into()); + return Err(EventProcessorError::InvalidEventLine { + message: format!("Invalid URI, must start with pubky://, {}", uri), + } + .into()); } // Extract the user_id from the initial part of the URI if let Some(user_id) = extract_segment(uri, "pubky://", "/pub/") { parsed_uri.user_id = PubkyId::try_from(user_id)?; } else { - return Err("Uri Pubky ID is invalid".into()); + return Err(EventProcessorError::InvalidEventLine { + message: format!("Uri Pubky ID is invalid, {}", uri), + } + .into()); } // Ensure that the URI belongs to pubky.app if let Some(app_segment) = extract_segment(uri, "/pub/", "/") { if app_segment != "pubky.app" { - return Err("The Event URI does not belong to pubky.app".into()); + return Err(EventProcessorError::InvalidEventLine { + message: format!("The Event URI does not belong to pubky.app, {}", uri), + } + .into()); } } else { - return Err("The Event URI is malformed".into()); + return Err(EventProcessorError::InvalidEventLine { + message: format!("The Event URI is malformed, {}", uri), + } + .into()); } // Extract post_id if present - if let Some(post_id) = extract_segment(uri, "/posts/", "/") { - parsed_uri.post_id = Some(post_id.to_string()); - } + parsed_uri.post_id = extract_segment(uri, "/posts/", "/") + .filter(|id| !id.is_empty()) + .map(String::from); // Extract follow_id if present - if let Some(follow_id) = extract_segment(uri, "/follows/", "/") { - parsed_uri.follow_id = Some(PubkyId::try_from(follow_id)?); - } + parsed_uri.follow_id = extract_segment(uri, "/follows/", "/") + .map(PubkyId::try_from) + .transpose() + .map_err(|e| EventProcessorError::InvalidEventLine { + message: format!("{}, {}", e, uri), + })?; // Extract muted_id if present - if let Some(muted_id) = extract_segment(uri, "/mutes/", "/") { - parsed_uri.muted_id = Some(PubkyId::try_from(muted_id)?); - } + parsed_uri.muted_id = extract_segment(uri, "/mutes/", "/") + .map(PubkyId::try_from) + .transpose() + .map_err(|e| EventProcessorError::InvalidEventLine { + message: format!("{}, {}", e, uri), + })?; // Extract bookmark_id if present - if let Some(bookmark_id) = extract_segment(uri, "/bookmarks/", "/") { - parsed_uri.bookmark_id = Some(bookmark_id.to_string()); - } + parsed_uri.bookmark_id = extract_segment(uri, "/bookmarks/", "/") + .filter(|id| !id.is_empty()) + .map(String::from); // Extract tag_id if present - if let Some(tag_id) = extract_segment(uri, "/tags/", "/") { - parsed_uri.tag_id = Some(tag_id.to_string()); - } + parsed_uri.tag_id = extract_segment(uri, "/tags/", "/") + .filter(|id| !id.is_empty()) + .map(String::from); // Extract file_id if present - if let Some(file_id) = extract_segment(uri, "/files/", "/") { - parsed_uri.file_id = Some(file_id.to_string()); - } + parsed_uri.file_id = extract_segment(uri, "/files/", "/") + .filter(|id| !id.is_empty()) + .map(String::from); Ok(parsed_uri) } @@ -83,3 +103,17 @@ fn extract_segment<'a>(uri: &'a str, start_pattern: &str, end_pattern: &str) -> Some(&uri[start_idx..end_idx]) } + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_bookmark_uri() { + let uri = + "pubky://phbhg3qgcttn95guepmbud1nzcxhg3xc5j5k4h7i8a4b6wb3nw1o/pub/pubky.app/bookmarks/"; + let parsed_uri = ParsedUri::try_from(uri).unwrap_or_default(); + println!("ParsedUri: {:?}", parsed_uri); + } +} From d46e9b01bc0785b89cc8f21dc4c21c09ca621834 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 22 Jan 2025 17:25:11 +0100 Subject: [PATCH 18/42] minor changes --- src/events/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/events/mod.rs b/src/events/mod.rs index 21207fea..66984855 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -95,7 +95,6 @@ impl Event { let uri = parts[1].to_string(); let parsed_uri = ParsedUri::try_from(uri.as_str())?; - println!("ParsedURI: {:?}", parsed_uri); //TODO: This conversion to a match statement that only uses IF conditions is silly. // We could be patter matching the split test for "posts", "follows", etc maybe? @@ -108,7 +107,7 @@ impl Event { post_id: parsed_uri .post_id .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing post_id, {} {}", parts[0], uri), + message: format!("Missing post_id, {}", line), })?, }, _ if uri.contains("/follows/") => ResourceType::Follow { @@ -116,7 +115,7 @@ impl Event { followee_id: parsed_uri .follow_id .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing followee_id, {} {}", parts[0], uri), + message: format!("Missing followee_id, {}", line), })?, }, _ if uri.contains("/mutes/") => ResourceType::Mute { @@ -124,14 +123,14 @@ impl Event { muted_id: parsed_uri .muted_id .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing muted_id, {} {}", parts[0], uri), + message: format!("Missing muted_id, {}", line), })?, }, _ if uri.contains("/bookmarks/") => ResourceType::Bookmark { user_id: parsed_uri.user_id, bookmark_id: parsed_uri.bookmark_id.ok_or( EventProcessorError::InvalidEventLine { - message: format!("Missing bookmark_id, {} {}", parts[0], uri), + message: format!("Missing bookmark_id, {}", line), }, )?, }, @@ -140,7 +139,7 @@ impl Event { tag_id: parsed_uri .tag_id .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing tag_id, {} {}", parts[0], uri), + message: format!("Missing tag_id, {}", line), })?, }, _ if uri.contains("/files/") => ResourceType::File { @@ -148,15 +147,16 @@ impl Event { file_id: parsed_uri .file_id .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing file_id, {} {}", parts[0], uri), + message: format!("Missing file_id, {}", line), })?, }, _ if uri.contains("/blobs") => return Ok(None), _ if uri.contains("/last_read") => return Ok(None), _ if uri.contains("/settings") => return Ok(None), + _ if uri.contains("/feeds") => return Ok(None), _ => { return Err(EventProcessorError::InvalidEventLine { - message: format!("Unrecognized resource in URI, {} {}", parts[0], uri), + message: format!("Unrecognized resource in URI, {}", line), } .into()) } From 22daacd28d651c5492284efcecdb05916cad4418 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 28 Jan 2025 11:23:47 +0100 Subject: [PATCH 19/42] add review changes --- benches/watcher.rs | 17 ++-- examples/from_file.rs | 15 ++-- src/events/handlers/bookmark.rs | 3 - src/events/handlers/follow.rs | 1 - src/events/handlers/mute.rs | 4 - src/events/handlers/tag.rs | 6 -- src/events/handlers/user.rs | 16 ++-- src/events/processor.rs | 21 ++--- src/events/retry/event.rs | 46 +---------- src/events/retry/manager.rs | 99 +++++++++-------------- src/events/uri.rs | 5 +- src/models/post/bookmark.rs | 1 - src/watcher.rs | 34 ++------ tests/watcher/bookmarks/retry_bookmark.rs | 6 -- tests/watcher/follows/retry_follow.rs | 7 -- tests/watcher/mutes/retry_mute.rs | 6 -- tests/watcher/posts/fail_reply.rs | 2 - tests/watcher/posts/fail_repost.rs | 2 - tests/watcher/posts/fail_user.rs | 2 - tests/watcher/posts/retry_all.rs | 6 -- tests/watcher/posts/retry_post.rs | 6 -- tests/watcher/posts/retry_reply.rs | 6 -- tests/watcher/posts/retry_repost.rs | 6 -- tests/watcher/tags/fail_index.rs | 2 - tests/watcher/tags/retry_post_tag.rs | 6 -- tests/watcher/tags/retry_user_tag.rs | 6 -- tests/watcher/utils/watcher.rs | 17 ++-- 27 files changed, 100 insertions(+), 248 deletions(-) diff --git a/benches/watcher.rs b/benches/watcher.rs index 4d564597..02843441 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -9,8 +9,8 @@ use pubky_nexus::{ EventProcessor, }; use setup::run_setup; -use std::time::Duration; -use tokio::{runtime::Runtime, sync::mpsc}; +use std::{sync::Arc, time::Duration}; +use tokio::runtime::Runtime; mod setup; @@ -30,13 +30,14 @@ async fn create_homeserver_with_events() -> (Testnet, String, SenderChannel) { let keypair = Keypair::random(); let user_id = keypair.public_key().to_z32(); - let retry_manager = RetryManager::initialise(mpsc::channel(1024)); - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = retry_manager.sender.clone(); + // Initializes a retry manager and ensures robustness by managing retries asynchronously + let (receiver_channel, sender_channel) = RetryManager::init_channels(); + // Create new asynchronous task to control the failed events - tokio::spawn(async move { - let _ = retry_manager.exec().await; - }); + RetryManager::process_messages(&receiver_channel).await; + + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = Arc::clone(&sender_channel); // Create and delete a user profile (as per your requirement) client diff --git a/examples/from_file.rs b/examples/from_file.rs index 0caebd58..0bfab18f 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -4,7 +4,7 @@ use pubky_nexus::{setup, types::DynError, Config, EventProcessor}; use std::fs::File; use std::io::{self, BufRead}; use std::path::Path; -use tokio::sync::mpsc; +use std::sync::Arc; // Create that file and add the file with that format // PUT homeserver_uri @@ -16,13 +16,14 @@ async fn main() -> Result<(), DynError> { let config = Config::from_env(); setup(&config).await; - let retry_manager = RetryManager::initialise(mpsc::channel(1024)); - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = retry_manager.sender.clone(); + // Initializes a retry manager and ensures robustness by managing retries asynchronously + let (receiver_channel, sender_channel) = RetryManager::init_channels(); + // Create new asynchronous task to control the failed events - tokio::spawn(async move { - let _ = retry_manager.exec().await; - }); + RetryManager::process_messages(&receiver_channel).await; + + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = Arc::clone(&sender_channel); let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 194560a6..72335cf0 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -38,7 +38,6 @@ pub async fn sync_put( match Bookmark::put_to_graph(&author_id, &post_id, &user_id, &id, indexed_at).await? { OperationOutcome::CreatedOrDeleted => false, OperationOutcome::Updated => true, - // TODO: Should return an error that should be processed by RetryManager OperationOutcome::MissingDependency => { let dependency = vec![format!("{author_id}:posts:{post_id}")]; return Err(EventProcessorError::MissingDependency { dependency }.into()); @@ -65,7 +64,6 @@ pub async fn del(user_id: PubkyId, bookmark_id: String) -> Result<(), DynError> } pub async fn sync_del(user_id: PubkyId, bookmark_id: String) -> Result<(), DynError> { - // DELETE FROM GRAPH let deleted_bookmark_info = Bookmark::del_from_graph(&user_id, &bookmark_id).await?; // Ensure the bookmark exists in the graph before proceeding let (post_id, author_id) = match deleted_bookmark_info { @@ -73,7 +71,6 @@ pub async fn sync_del(user_id: PubkyId, bookmark_id: String) -> Result<(), DynEr None => return Err(EventProcessorError::SkipIndexing.into()), }; - // DELETE FROM INDEXes Bookmark::del_from_index(&user_id, &post_id, &author_id).await?; // Update user counts UserCounts::update(&user_id, "bookmarks", JsonAction::Decrement(1)).await?; diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index 0472d731..2b9ec4c0 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -66,7 +66,6 @@ pub async fn del(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), DynEr } pub async fn sync_del(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), DynError> { - // DELETE FROM GRAPH match Followers::del_from_graph(&follower_id, &followee_id).await? { // Both users exists but they do not have that relationship OperationOutcome::Updated => Ok(()), diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index 35d2a8ff..9c0a7985 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -15,7 +15,6 @@ pub async fn put(user_id: PubkyId, muted_id: PubkyId, _blob: &[u8]) -> Result<() } pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { - // SAVE TO GRAPH // (user_id)-[:MUTED]->(muted_id) match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), @@ -24,7 +23,6 @@ pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro Err(EventProcessorError::MissingDependency { dependency }.into()) } OperationOutcome::CreatedOrDeleted => { - // SAVE TO INDEX Muted(vec![muted_id.to_string()]) .put_to_index(&user_id) .await @@ -38,12 +36,10 @@ pub async fn del(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { } pub async fn sync_del(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { - // DELETE FROM GRAPH match Muted::del_from_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), OperationOutcome::MissingDependency => Err(EventProcessorError::SkipIndexing.into()), OperationOutcome::CreatedOrDeleted => { - // REMOVE FROM INDEX Muted(vec![muted_id.to_string()]) .del_from_index(&user_id) .await diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 731e9efc..b8bf527b 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -67,7 +67,6 @@ async fn put_sync_post( post_uri: String, indexed_at: i64, ) -> Result<(), DynError> { - // SAVE TO GRAPH match TagPost::put_to_graph( &tagger_user_id, &author_id, @@ -146,7 +145,6 @@ async fn put_sync_user( tag_label: String, indexed_at: i64, ) -> Result<(), DynError> { - // SAVE TO GRAPH match TagUser::put_to_graph( &tagger_user_id, &tagged_user_id, @@ -193,8 +191,6 @@ async fn put_sync_user( pub async fn del(user_id: PubkyId, tag_id: String) -> Result<(), DynError> { debug!("Deleting tag: {} -> {}", user_id, tag_id); - // DELETE FROM GRAPH - // Maybe better if we add as a local function instead of part of the trait? let tag_details = TagUser::del_from_graph(&user_id, &tag_id).await?; // CHOOSE THE EVENT TYPE if let Some((tagged_user_id, post_id, author_id, label)) = tag_details { @@ -213,8 +209,6 @@ pub async fn del(user_id: PubkyId, tag_id: String) -> Result<(), DynError> { } } } else { - // TODO: Should return an error that should be processed by RetryManager - // WIP: Create a custom error type to pass enough info to the RetryManager return Err(EventProcessorError::SkipIndexing.into()); } Ok(()) diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index 37d5e19a..2991aa64 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -24,16 +24,12 @@ pub async fn put(user_id: PubkyId, blob: &[u8]) -> Result<(), DynError> { pub async fn sync_put(user: PubkyAppUser, user_id: PubkyId) -> Result<(), DynError> { // Create UserDetails object let user_details = UserDetails::from_homeserver(user, &user_id).await?; - // SAVE TO GRAPH - match user_details.put_to_graph().await { - Ok(_) => (), - Err(e) => { - return Err(EventProcessorError::GraphQueryFailed { - message: format!("{:?}", e), - } - .into()) - } - } + user_details + .put_to_graph() + .await + .map_err(|e| EventProcessorError::GraphQueryFailed { + message: format!("{:?}", e), + })?; // SAVE TO INDEX let user_id = user_details.id.clone(); UserSearch::put_to_index(&[&user_details]).await?; diff --git a/src/events/processor.rs b/src/events/processor.rs index 92a71fd3..0d49b2ac 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,5 +1,5 @@ use super::error::EventProcessorError; -use super::retry::manager::{SenderChannel, SenderMessage}; +use super::retry::manager::{RetryQueueMessage, SenderChannel}; use super::Event; use crate::events::retry::event::RetryEvent; use crate::types::DynError; @@ -11,11 +11,14 @@ use log::{debug, error, info}; pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, - pub sender: SenderChannel, + pub retry_manager_sender_channel: SenderChannel, } impl EventProcessor { - pub async fn from_config(config: &Config, tx: SenderChannel) -> Result { + pub async fn from_config( + config: &Config, + retry_manager_sender_channel: SenderChannel, + ) -> Result { let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; @@ -27,7 +30,7 @@ impl EventProcessor { Ok(Self { homeserver, limit, - sender: tx, + retry_manager_sender_channel, }) } @@ -43,13 +46,13 @@ impl EventProcessor { /// # Parameters /// - `homeserver_id`: A `String` representing the URL of the homeserver to be used in the test environment. /// - `tx`: A `SenderChannel` used to handle outgoing messages or events. - pub async fn test(homeserver_id: String, tx: SenderChannel) -> Self { + pub async fn test(homeserver_id: String, retry_manager_sender_channel: SenderChannel) -> Self { let id = PubkyId(homeserver_id.to_string()); let homeserver = Homeserver::new(id).await.unwrap(); Self { homeserver, limit: 1000, - sender: tx, + retry_manager_sender_channel, } } @@ -158,9 +161,9 @@ impl EventProcessor { let index_key = format!("{}:{}", event.event_type, index); // Send event to the retry manager - let sender = self.sender.lock().await; - match sender - .send(SenderMessage::ProcessEvent(index_key, retry_event)) + let sender_channel = self.retry_manager_sender_channel.lock().await; + match sender_channel + .send(RetryQueueMessage::ProcessEvent(index_key, retry_event)) .await { Ok(_) => { diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index acf6d1a6..162ced5a 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -2,11 +2,7 @@ use async_trait::async_trait; use chrono::Utc; use serde::{Deserialize, Serialize}; -use crate::{ - events::{error::EventProcessorError, EventType}, - types::DynError, - RedisOps, -}; +use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; @@ -68,15 +64,6 @@ impl RetryEvent { } } - /// Generates the opposite event line by replacing the event type, effectively - /// converting delete operations into put operations - /// - /// # Arguments - /// * `event_line` - A string slice representing the event line. - pub fn generate_opposite_event_line(event_line: &str) -> String { - event_line.replace(&EventType::Del.to_string(), &EventType::Put.to_string()) - } - /// Stores an event in both a sorted set and a JSON index in Redis. /// It adds an event line to a Redis sorted set with a timestamp-based score /// and also stores the event details in a separate JSON index for retrieval. @@ -102,44 +89,19 @@ impl RetryEvent { /// # Arguments /// * `event_index` - A `&str` representing the event index to check pub async fn check_uri(event_index: &str) -> Result, DynError> { - if let Some(post_details) = Self::check_sorted_set_member( + Self::check_sorted_set_member( Some(RETRY_MAMAGER_PREFIX), &RETRY_MANAGER_EVENTS_INDEX, &[event_index], ) - .await? - { - return Ok(Some(post_details)); - } - Ok(None) + .await } /// Retrieves an event from the JSON index in Redis based on its index /// # Arguments /// * `event_index` - A `&str` representing the event index to retrieve pub async fn get_from_index(event_index: &str) -> Result, DynError> { - let mut found_event = None; let index = &[RETRY_MANAGER_STATE_INDEX, [event_index]].concat(); - if let Some(fail_event) = Self::try_from_index_json(index).await? { - found_event = Some(fail_event); - } - Ok(found_event) - } - - /// Deletes event-related indices from the retry queue in Redis - /// # Arguments - /// - `event_line` - A `String` representing the event line that needs to be unindexed from the retry queue - pub async fn delete(&self, event_line: String) -> Result<(), DynError> { - let put_event_line = Self::generate_opposite_event_line(&event_line); - Self::remove_from_index_sorted_set( - Some(RETRY_MAMAGER_PREFIX), - &RETRY_MANAGER_EVENTS_INDEX, - &[&put_event_line], - ) - .await?; - - let index = &[RETRY_MANAGER_STATE_INDEX, [&put_event_line]].concat(); - Self::remove_from_index_multiple_json(&[index]).await?; - Ok(()) + Self::try_from_index_json(index).await } } diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index 9a5463a1..9859b695 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -1,19 +1,18 @@ -use log::error; +use log::{debug, error}; use std::sync::Arc; +use tokio::sync::mpsc; use tokio::sync::{ mpsc::{Receiver, Sender}, Mutex, }; -use crate::types::DynError; - use super::event::RetryEvent; pub const CHANNEL_BUFFER: usize = 1024; /// Represents commands sent to the retry manager for handling events in the retry queue #[derive(Debug, Clone)] -pub enum SenderMessage { +pub enum RetryQueueMessage { // Command to retry all pending events that are waiting to be indexed RetryEvent(String), /// Command to process a specific event in the retry queue @@ -23,72 +22,52 @@ pub enum SenderMessage { ProcessEvent(String, RetryEvent), } -pub type SenderChannel = Arc>>; -type ReceiverChannel = Arc>>; - -pub struct RetryManager { - pub sender: SenderChannel, - receiver: ReceiverChannel, -} +pub type SenderChannel = Arc>>; +type ReceiverChannel = Arc>>; -/// Initializes a new `RetryManager` with a message-passing channel. +/// Manages the retry queue for processing failed events. /// -/// This function sets up the `RetryManager` by taking a tuple containing -/// a `Sender` and `Receiver` from a multi-producer, single-consumer (mpsc) channel. +/// The `RetryManager` is responsible for handling messages that involve retrying or +/// processing failed events in an asynchronous event system. It listens for incoming +/// retry-related messages and processes them accordingly /// -/// # Parameters -/// - `(tx, rx)`: A tuple containing: -/// - `tx` (`Sender`): The sending half of the mpsc channel, used to dispatch messages to the manager. -/// - `rx` (`Receiver`): The receiving half of the mpsc channel, used to listen for incoming messages. +/// ## Responsibilities: +/// - Receives and processes messages related to event retries +/// - Maintains a queue of events that need to be retried +/// - Ensures failed events are reprocessed in an orderly manner +pub struct RetryManager {} + +/// Initializes a new `RetryManager` with a message-passing channel impl RetryManager { - pub fn initialise((tx, rx): (Sender, Receiver)) -> Self { - Self { - receiver: Arc::new(Mutex::new(rx)), - sender: Arc::new(Mutex::new(tx)), - } + pub fn init_channels() -> (ReceiverChannel, SenderChannel) { + let (tx, rx) = mpsc::channel(CHANNEL_BUFFER); + ( + Arc::new(Mutex::new(rx)), // Receiver channel + Arc::new(Mutex::new(tx)), // Sender channel + ) } /// Executes the main event loop to process messages from the channel /// This function listens for incoming messages on the receiver channel and handles them - /// based on their type: - /// - **`SenderMessage::Retry`**: Triggers the retry process - /// - **`SenderMessage::Add`**: Queues a failed event in the retry cache for future processing - /// + /// based on their type /// The loop runs continuously until the channel is closed, ensuring that all messages - /// are processed appropriately. - pub async fn exec(&self) -> Result<(), DynError> { - let mut rx = self.receiver.lock().await; - // Listen all the messages in the channel - while let Some(message) = rx.recv().await { - match message { - SenderMessage::RetryEvent(homeserver_pubky) => { - self.retry_events_for_homeserver(&homeserver_pubky); - } - SenderMessage::ProcessEvent(index_key, retry_event) => { - self.queue_failed_event(index_key, retry_event).await?; + /// are processed appropriately + pub async fn process_messages(receiver_channel: &ReceiverChannel) { + let receiver_channel = Arc::clone(receiver_channel); + tokio::spawn(async move { + let mut rx = receiver_channel.lock().await; + // Listen all the messages in the channel + while let Some(message) = rx.recv().await { + match message { + RetryQueueMessage::ProcessEvent(index_key, retry_event) => { + error!("{}, {}", retry_event.error_type, index_key); + if let Err(err) = retry_event.put_to_index(index_key).await { + error!("Failed to put event to index: {}", err); + } + } + _ => debug!("New message received, not handled: {:?}", message), } } - } - Ok(()) - } - - fn retry_events_for_homeserver(&self, _homeserver_pubky: &str) { - // WIP: Retrieve the homeserver events from the SORTED SET - // RetryManager:events - } - - /// Stores the event line in the Redis cache, adding it to the retry queue for - /// future processing - /// # Arguments - /// - `index_key`: A `String` representing the compacted key for the event to be stored in Redis - /// - `retry_event`: A `RetryEvent` instance containing the details of the failed event - async fn queue_failed_event( - &self, - index_key: String, - retry_event: RetryEvent, - ) -> Result<(), DynError> { - error!("{}, {}", retry_event.error_type, index_key); - retry_event.put_to_index(index_key).await?; - Ok(()) + }); } } diff --git a/src/events/uri.rs b/src/events/uri.rs index 82623567..52ecf344 100644 --- a/src/events/uri.rs +++ b/src/events/uri.rs @@ -114,6 +114,9 @@ mod tests { let uri = "pubky://phbhg3qgcttn95guepmbud1nzcxhg3xc5j5k4h7i8a4b6wb3nw1o/pub/pubky.app/bookmarks/"; let parsed_uri = ParsedUri::try_from(uri).unwrap_or_default(); - println!("ParsedUri: {:?}", parsed_uri); + assert_eq!( + parsed_uri.bookmark_id, None, + "The provided URI has bookmark_id" + ); } } diff --git a/src/models/post/bookmark.rs b/src/models/post/bookmark.rs index 85950473..1bbaa7a9 100644 --- a/src/models/post/bookmark.rs +++ b/src/models/post/bookmark.rs @@ -159,7 +159,6 @@ impl Bookmark { return Ok(Some((post_id, author_id))); } } - // No rows matched the query. The pattern did not yield any results Ok(None) } diff --git a/src/watcher.rs b/src/watcher.rs index eda83e23..f47aa00a 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -1,14 +1,11 @@ use log::error; use log::info; -use pubky_nexus::events::retry::manager::SenderMessage; -use pubky_nexus::events::retry::manager::{RetryManager, CHANNEL_BUFFER}; +use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::PubkyConnector; use pubky_nexus::{setup, Config, EventProcessor}; -use tokio::sync::mpsc; +use std::sync::Arc; use tokio::time::{sleep, Duration}; -const RETRY_THRESHOLD: u8 = 5; - /// Watches over a homeserver `/events` and writes into the Nexus databases #[tokio::main] async fn main() -> Result<(), Box> { @@ -20,22 +17,17 @@ async fn main() -> Result<(), Box> { PubkyConnector::initialise(&config, None)?; // Initializes a retry manager and ensures robustness by managing retries asynchronously - let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = retry_manager.sender.clone(); + let (receiver_channel, sender_channel) = RetryManager::init_channels(); + // Create new asynchronous task to control the failed events - // TODO: Assure the thread safety - tokio::spawn(async move { - let _ = retry_manager.exec().await; - }); + RetryManager::process_messages(&receiver_channel).await; + + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = Arc::clone(&sender_channel); // Create and configure the event processor let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; - // Experimental. We need to think how/where achieve that - // Maybe add in .env file... - let mut retry_failed_events = 0; - loop { info!("Fetching events..."); if let Err(e) = event_processor.run().await { @@ -43,15 +35,5 @@ async fn main() -> Result<(), Box> { } // Wait for X milliseconds before fetching events again sleep(Duration::from_millis(config.watcher_sleep)).await; - retry_failed_events += 1; - - if RETRY_THRESHOLD == retry_failed_events { - let sender = event_processor.sender.lock().await; - let _ = sender - .send(SenderMessage::RetryEvent( - event_processor.homeserver.id.to_string(), - )) - .await; - } } } diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index 820678c2..d6d1cfbc 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the post to connect the bookmark -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_bookmark_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -51,11 +49,9 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&put_index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -82,11 +78,9 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 7e00d619..892f02ed 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the followee to connect with follower -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_follow_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -29,7 +27,6 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { }; let follower_id = test.create_user(&follower_keypair, &follower_user).await?; - // Mute the user test.create_follow(&follower_id, &followee_id).await?; tokio::time::sleep(Duration::from_millis(500)).await; @@ -41,11 +38,9 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&follow_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -72,11 +67,9 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&follow_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 09f8e47f..496225f3 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the mutee to connect with muter -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_mute_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -41,11 +39,9 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&mute_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -72,11 +68,9 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&mute_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/posts/fail_reply.rs b/tests/watcher/posts/fail_reply.rs index 350fa233..bd0c6429 100644 --- a/tests/watcher/posts/fail_reply.rs +++ b/tests/watcher/posts/fail_reply.rs @@ -6,8 +6,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::types::DynError; /// The user profile is stored in the homeserver and synched in the graph, but the posts just exist in the homeserver -// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), -// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_reply_without_post_parent() -> Result<(), DynError> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/posts/fail_repost.rs b/tests/watcher/posts/fail_repost.rs index 6a69b0f9..8c6236a4 100644 --- a/tests/watcher/posts/fail_repost.rs +++ b/tests/watcher/posts/fail_repost.rs @@ -6,8 +6,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::types::DynError; /// The user profile is stored in the homeserver and synched in the graph, but the posts just exist in the homeserver -// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), -// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_repost_without_post_parent() -> Result<(), DynError> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/posts/fail_user.rs b/tests/watcher/posts/fail_user.rs index ab54009b..ceade2ec 100644 --- a/tests/watcher/posts/fail_user.rs +++ b/tests/watcher/posts/fail_user.rs @@ -4,8 +4,6 @@ use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind}; use pubky_common::crypto::Keypair; /// The user profile is stored in the homeserver. Missing the author to connect the post -// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), -// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_without_user() -> Result<()> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index 767d80b2..0f0f67b6 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the post to connect the new one -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -54,11 +52,9 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_reply_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -90,11 +86,9 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_reply_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index caf708ec..88cab15d 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the author to connect the post -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -39,11 +37,9 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&post_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -73,11 +69,9 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&post_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 1ffe1531..5b6c03f0 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the post to connect the new one -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_reply_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -49,11 +47,9 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&reply_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -81,11 +77,9 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&reply_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index aed853d4..f7097e2a 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -7,8 +7,6 @@ use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; /// The user profile is stored in the homeserver. Missing the post to connect the new one -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_repost_cannot_index() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -52,11 +50,9 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -84,11 +80,9 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/tags/fail_index.rs b/tests/watcher/tags/fail_index.rs index 829d5742..8be9c966 100644 --- a/tests/watcher/tags/fail_index.rs +++ b/tests/watcher/tags/fail_index.rs @@ -5,8 +5,6 @@ use log::error; use pubky_app_specs::{traits::HashId, PubkyAppPost, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; -// These types of tests (e.g., fail_xxxx) can be used to test the graph queries (PUT/DEL), -// as they do not rely on the `WatcherTest` event processor #[tokio_shared_rt::test(shared)] async fn test_homeserver_tag_cannot_add_while_index() -> Result<()> { let mut test = WatcherTest::setup().await?; diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 75267221..d1dd6a52 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -7,8 +7,6 @@ use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -65,11 +63,9 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -98,11 +94,9 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 3774407e..35255c2e 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -7,8 +7,6 @@ use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; -// These types of tests (e.g., retry_xxxx) can be used to verify whether the `RetryManager` -// cache correctly adds the events as expected. #[tokio_shared_rt::test(shared)] async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { let mut test = WatcherTest::setup().await?; @@ -56,11 +54,9 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); - // Assert if the event is in the timeline let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state hash map let event_retry = RetryEvent::get_from_index(&index_key).await.unwrap(); assert!(event_retry.is_some()); @@ -88,11 +84,9 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); - // Assert that the event does not exist in the sorted set. In that case PUT event let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); - // Assert if the event is in the state. JSON let event_retry = RetryEvent::get_from_index(&del_index_key).await.unwrap(); assert!(event_retry.is_some()); diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index 580b7dfa..d84bbb74 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use super::dht::TestnetDHTNetwork; use anyhow::{anyhow, Result}; use chrono::Utc; @@ -7,11 +9,10 @@ use pubky_app_specs::{ }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; -use pubky_nexus::events::retry::manager::{RetryManager, CHANNEL_BUFFER}; +use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::events::Event; use pubky_nexus::types::DynError; use pubky_nexus::{setup, Config, EventProcessor, PubkyConnector}; -use tokio::sync::mpsc; /// Struct to hold the setup environment for tests pub struct WatcherTest { @@ -46,12 +47,14 @@ impl WatcherTest { let homeserver = Homeserver::start_test(&testnet).await?; let homeserver_id = homeserver.public_key().to_string(); - let retry_manager = RetryManager::initialise(mpsc::channel(CHANNEL_BUFFER)); - let sender_clone = retry_manager.sender.clone(); + // Initializes a retry manager and ensures robustness by managing retries asynchronously + let (receiver_channel, sender_channel) = RetryManager::init_channels(); + + // Create new asynchronous task to control the failed events + RetryManager::process_messages(&receiver_channel).await; - tokio::spawn(async move { - let _ = retry_manager.exec().await; - }); + // Prepare the sender channel to send the messages to the retry manager + let sender_clone = Arc::clone(&sender_channel); match PubkyConnector::initialise(&config, Some(&testnet)) { Ok(_) => debug!("WatcherTest: PubkyConnector initialised"), From 2dd10b8e691a0c58de27ad97c790a63fd2b04276 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 28 Jan 2025 12:31:14 +0100 Subject: [PATCH 20/42] fix error --- src/models/user/search.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/user/search.rs b/src/models/user/search.rs index 5030da23..ebe0e1a5 100644 --- a/src/models/user/search.rs +++ b/src/models/user/search.rs @@ -124,6 +124,7 @@ impl UserSearch { } Self::remove_from_index_sorted_set( + None, &USER_NAME_KEY_PARTS, records_to_delete .iter() From d3150f5536e73a28c25aee71e08980bbad440de0 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 28 Jan 2025 14:24:37 +0100 Subject: [PATCH 21/42] improve pubky connector --- src/db/connectors/pubky.rs | 84 ++++++++++++++-------------------- src/watcher.rs | 2 +- tests/watcher/utils/watcher.rs | 2 +- 3 files changed, 36 insertions(+), 52 deletions(-) diff --git a/src/db/connectors/pubky.rs b/src/db/connectors/pubky.rs index 690c91e3..98ea92c9 100644 --- a/src/db/connectors/pubky.rs +++ b/src/db/connectors/pubky.rs @@ -2,76 +2,60 @@ use crate::Config; use mainline::Testnet; use pubky::Client; use std::sync::Arc; +use thiserror::Error; use tokio::sync::OnceCell; -static PUBKY_CONNECTOR_SINGLETON: OnceCell = OnceCell::const_new(); +static PUBKY_CONNECTOR_SINGLETON: OnceCell> = OnceCell::const_new(); -#[derive(Debug, Clone)] -pub struct PubkyConnector { - pubky_client: Arc, -} - -#[derive(Debug)] +#[derive(Debug, Error)] pub enum PubkyConnectorError { + #[error("PubkyConnector has already been initialized")] AlreadyInitialized, + + #[error("PubkyConnector not initialized")] NotInitialized, - IoError(std::io::Error), -} -impl From for PubkyConnectorError { - fn from(e: std::io::Error) -> Self { - PubkyConnectorError::IoError(e) - } -} + #[error("I/O error: {0}")] + IoError(#[from] std::io::Error), -impl std::fmt::Display for PubkyConnectorError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PubkyConnectorError::AlreadyInitialized => { - write!(f, "PubkyConnector has already been initialized") - } - PubkyConnectorError::NotInitialized => write!(f, "PubkyConnector not initialized"), - PubkyConnectorError::IoError(e) => write!(f, "I/O error: {}", e), - } - } + #[error("Client initialization error: {0}")] + ClientError(String), } -impl std::error::Error for PubkyConnectorError {} +pub struct PubkyConnector; impl PubkyConnector { /// Initializes the PubkyConnector singleton with the given configuration - pub fn initialise( + pub async fn initialise( config: &Config, testnet: Option<&Testnet>, ) -> Result<(), PubkyConnectorError> { - // There is not need to initialise, already in the global context - if PUBKY_CONNECTOR_SINGLETON.get().is_some() { - return Ok(()); - } - let pubky_client = match testnet { - Some(testnet) => Client::builder().testnet(testnet).build()?, - None => match config.testnet { - true => Client::testnet()?, - false => Client::new()?, - }, - }; - - let manager = Self { - pubky_client: Arc::new(pubky_client), - }; - PUBKY_CONNECTOR_SINGLETON - .set(manager) - .map_err(|_| PubkyConnectorError::AlreadyInitialized)?; - Ok(()) + .get_or_try_init(|| async { + let pubky_client = match testnet { + Some(testnet) => Client::builder() + .testnet(testnet) + .build() + .map_err(|e| PubkyConnectorError::ClientError(e.to_string()))?, + None => match config.testnet { + true => Client::testnet() + .map_err(|e| PubkyConnectorError::ClientError(e.to_string()))?, + false => Client::new() + .map_err(|e| PubkyConnectorError::ClientError(e.to_string()))?, + }, + }; + + Ok(Arc::new(pubky_client)) + }) + .await + .map(|_| ()) } /// Retrieves the shared Client connection. pub fn get_pubky_client() -> Result, PubkyConnectorError> { - if let Some(resolver) = PUBKY_CONNECTOR_SINGLETON.get() { - Ok(resolver.pubky_client.clone()) - } else { - Err(PubkyConnectorError::NotInitialized) - } + PUBKY_CONNECTOR_SINGLETON + .get() + .cloned() + .ok_or(PubkyConnectorError::NotInitialized) } } diff --git a/src/watcher.rs b/src/watcher.rs index f47aa00a..769803f7 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -14,7 +14,7 @@ async fn main() -> Result<(), Box> { setup(&config).await; // Initializes the PubkyConnector with the configuration - PubkyConnector::initialise(&config, None)?; + PubkyConnector::initialise(&config, None).await?; // Initializes a retry manager and ensures robustness by managing retries asynchronously let (receiver_channel, sender_channel) = RetryManager::init_channels(); diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index d84bbb74..ad203757 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -56,7 +56,7 @@ impl WatcherTest { // Prepare the sender channel to send the messages to the retry manager let sender_clone = Arc::clone(&sender_channel); - match PubkyConnector::initialise(&config, Some(&testnet)) { + match PubkyConnector::initialise(&config, Some(&testnet)).await { Ok(_) => debug!("WatcherTest: PubkyConnector initialised"), Err(e) => debug!("WatcherTest: {}", e), } From 53c74dff748973cff5da951f27883a97b8149523 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 28 Jan 2025 14:48:27 +0100 Subject: [PATCH 22/42] fix test errors --- tests/watcher/bookmarks/retry_bookmark.rs | 4 ++-- tests/watcher/follows/retry_follow.rs | 4 ++-- tests/watcher/mutes/retry_mute.rs | 4 ++-- tests/watcher/posts/retry_all.rs | 4 ++-- tests/watcher/posts/retry_post.rs | 4 ++-- tests/watcher/posts/retry_reply.rs | 4 ++-- tests/watcher/posts/retry_repost.rs | 4 ++-- tests/watcher/tags/retry_post_tag.rs | 4 ++-- tests/watcher/tags/retry_user_tag.rs | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index d6d1cfbc..218169b0 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -65,7 +65,7 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { assert_eq!(dependency.len(), 1); assert_eq!(dependency[0], dependency_uri); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; // DEL bookmark @@ -90,7 +90,7 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 892f02ed..021c031f 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -55,7 +55,7 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { assert_eq!(dependency.len(), 1); assert_eq!(dependency[0], dependency_uri) } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&follow_url).await?; @@ -79,7 +79,7 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 496225f3..2dc83dc0 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -56,7 +56,7 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { assert_eq!(dependency.len(), 1); assert_eq!(dependency[0], dependency_uri) } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&mute_url).await?; @@ -80,7 +80,7 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index 0f0f67b6..7f91a4f6 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -74,7 +74,7 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&repost_reply_url).await?; @@ -98,7 +98,7 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index 88cab15d..710dde62 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -57,7 +57,7 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&dependency_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&post_url).await?; @@ -81,7 +81,7 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 5b6c03f0..ea87863a 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -65,7 +65,7 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&dependency_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&reply_url).await?; @@ -89,7 +89,7 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index f7097e2a..ad3d12e6 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -68,7 +68,7 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&dependency_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&repost_url).await?; @@ -92,7 +92,7 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index d1dd6a52..1953c997 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -82,7 +82,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&dependency_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&tag_url).await?; @@ -106,7 +106,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 35255c2e..c673fb8f 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -72,7 +72,7 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&dependency_uri).unwrap() ); } - _ => assert!(false, "The error type has to be MissingDependency type"), + _ => panic!("The error type has to be MissingDependency type"), }; test.del(&tag_url).await?; @@ -96,7 +96,7 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { match event_state.error_type { EventProcessorError::SkipIndexing => (), - _ => assert!(false, "The error type has to be SkipIndexing type"), + _ => panic!("The error type has to be SkipIndexing type"), }; Ok(()) From a453f1693722665bd2af623d02bdf32d66ecbd0c Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 29 Jan 2025 16:36:34 +0100 Subject: [PATCH 23/42] test the retry mananger tests without sleep --- tests/watcher/bookmarks/retry_bookmark.rs | 4 ++-- tests/watcher/follows/retry_follow.rs | 4 ++-- tests/watcher/mutes/retry_mute.rs | 4 ++-- tests/watcher/posts/retry_all.rs | 4 ++-- tests/watcher/posts/retry_post.rs | 4 ++-- tests/watcher/posts/retry_reply.rs | 4 ++-- tests/watcher/posts/retry_repost.rs | 4 ++-- tests/watcher/tags/retry_post_tag.rs | 4 ++-- tests/watcher/tags/retry_user_tag.rs | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index 218169b0..4c02e8da 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -41,7 +41,7 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { ); // PUT bookmark test.put(&bookmark_url, bookmark).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let put_index_key = format!( "{}:{}", @@ -70,7 +70,7 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { // DEL bookmark test.del(&bookmark_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 021c031f..19a9c2b5 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -28,7 +28,7 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { let follower_id = test.create_user(&follower_keypair, &follower_user).await?; test.create_follow(&follower_id, &followee_id).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let follow_url = format!("pubky://{follower_id}/pub/pubky.app/follows/{followee_id}"); @@ -59,7 +59,7 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { }; test.del(&follow_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 2dc83dc0..5f90bb99 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -29,7 +29,7 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { // Mute the user test.create_mute(&muter_id, &mutee_id).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let mute_url = format!("pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}"); @@ -60,7 +60,7 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { }; test.del(&mute_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index 7f91a4f6..4c15142e 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -42,7 +42,7 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { }; let repost_reply_post_id = test.create_post(&user_id, &repost_reply_post).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let repost_reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}"); @@ -78,7 +78,7 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { }; test.del(&repost_reply_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index 710dde62..fb88836f 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -27,7 +27,7 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { }; let post_id = test.create_post(&user_id, &post).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let post_url = format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}"); @@ -61,7 +61,7 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { }; test.del(&post_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index ea87863a..2a9df08a 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -37,7 +37,7 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { }; let reply_id = test.create_post(&user_id, &reply_post).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{reply_id}"); @@ -69,7 +69,7 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { }; test.del(&reply_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index ad3d12e6..4b5d0097 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -40,7 +40,7 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { }; let repost_id = test.create_post(&user_id, &repost_post).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let repost_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_id}"); @@ -72,7 +72,7 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { }; test.del(&repost_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index 1953c997..b031a3e4 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -55,7 +55,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { // That operation is going to write the event in the pending events queue, so block a bit the thread // to let write the indexes test.put(tag_url.as_str(), tag).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( "{}:{}", @@ -86,7 +86,7 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { }; test.del(&tag_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index c673fb8f..abc5f591 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -46,7 +46,7 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { // That operation is going to write the event in the pending events queue, so block a bit the thread // to let write the indexes test.put(tag_url.as_str(), tag).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( "{}:{}", @@ -76,7 +76,7 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { }; test.del(&tag_url).await?; - tokio::time::sleep(Duration::from_millis(500)).await; + //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", From f27ddd69807249fe0b8e379d1aaa4874fdae3a05 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 29 Jan 2025 17:13:51 +0100 Subject: [PATCH 24/42] delete the tokio::sleep and add a util --- tests/watcher/bookmarks/retry_bookmark.rs | 7 +++-- tests/watcher/follows/retry_follow.rs | 4 --- tests/watcher/mutes/retry_mute.rs | 4 --- tests/watcher/posts/retry_all.rs | 4 --- tests/watcher/posts/retry_post.rs | 4 --- tests/watcher/posts/retry_reply.rs | 4 --- tests/watcher/posts/retry_repost.rs | 4 --- tests/watcher/tags/retry_post_tag.rs | 4 --- tests/watcher/tags/retry_user_tag.rs | 4 --- tests/watcher/utils/watcher.rs | 33 ++++++++++++++++++++++- 10 files changed, 35 insertions(+), 37 deletions(-) diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index 4c02e8da..d7af4e1c 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -1,6 +1,4 @@ -use std::time::Duration; - -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{try_until_write, WatcherTest}; use anyhow::Result; use pubky_app_specs::{traits::HashId, PubkyAppBookmark, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -41,7 +39,6 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { ); // PUT bookmark test.put(&bookmark_url, bookmark).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let put_index_key = format!( "{}:{}", @@ -49,6 +46,8 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); + try_until_write(&put_index_key).await; + let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 19a9c2b5..057d357b 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::PubkyAppUser; @@ -28,7 +26,6 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { let follower_id = test.create_user(&follower_keypair, &follower_user).await?; test.create_follow(&follower_id, &followee_id).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let follow_url = format!("pubky://{follower_id}/pub/pubky.app/follows/{followee_id}"); @@ -59,7 +56,6 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { }; test.del(&follow_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 5f90bb99..661e02db 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::PubkyAppUser; @@ -29,7 +27,6 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { // Mute the user test.create_mute(&muter_id, &mutee_id).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let mute_url = format!("pubky://{muter_id}/pub/pubky.app/mutes/{mutee_id}"); @@ -60,7 +57,6 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { }; test.del(&mute_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index 4c15142e..a639ec47 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; @@ -42,7 +40,6 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { }; let repost_reply_post_id = test.create_post(&user_id, &repost_reply_post).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let repost_reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_reply_post_id}"); @@ -78,7 +75,6 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { }; test.del(&repost_reply_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index fb88836f..aaaa61e8 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind}; @@ -27,7 +25,6 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { }; let post_id = test.create_post(&user_id, &post).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let post_url = format!("pubky://{user_id}/pub/pubky.app/posts/{post_id}"); @@ -61,7 +58,6 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { }; test.del(&post_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 2a9df08a..34427431 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind, PubkyAppUser}; @@ -37,7 +35,6 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { }; let reply_id = test.create_post(&user_id, &reply_post).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let reply_url = format!("pubky://{user_id}/pub/pubky.app/posts/{reply_id}"); @@ -69,7 +66,6 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { }; test.del(&reply_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index 4b5d0097..17214734 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; @@ -40,7 +38,6 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { }; let repost_id = test.create_post(&user_id, &repost_post).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let repost_url = format!("pubky://{user_id}/pub/pubky.app/posts/{repost_id}"); @@ -72,7 +69,6 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { }; test.del(&repost_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index b031a3e4..f563db12 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use chrono::Utc; @@ -55,7 +53,6 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { // That operation is going to write the event in the pending events queue, so block a bit the thread // to let write the indexes test.put(tag_url.as_str(), tag).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( "{}:{}", @@ -86,7 +83,6 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { }; test.del(&tag_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index abc5f591..368447b9 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use chrono::Utc; @@ -46,7 +44,6 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { // That operation is going to write the event in the pending events queue, so block a bit the thread // to let write the indexes test.put(tag_url.as_str(), tag).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let index_key = format!( "{}:{}", @@ -76,7 +73,6 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { }; test.del(&tag_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index ad203757..2778b75d 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -1,14 +1,16 @@ use std::sync::Arc; +use std::time::Duration; use super::dht::TestnetDHTNetwork; use anyhow::{anyhow, Result}; use chrono::Utc; -use log::debug; +use log::{debug, info}; use pubky_app_specs::{ traits::TimestampId, PubkyAppFile, PubkyAppFollow, PubkyAppPost, PubkyAppUser, }; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; +use pubky_nexus::events::retry::event::RetryEvent; use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::events::Event; use pubky_nexus::types::DynError; @@ -268,3 +270,32 @@ pub async fn retrieve_and_handle_event_line(event_line: &str) -> Result<(), DynE Ok(()) } + +/// Attempts to read an event index with retries before timing out +/// # Arguments +/// * `event_index` - A string slice representing the index to check +pub async fn try_until_write(event_index: &str) { + const SLEEP_MS: u64 = 20; + const MAX_RETRIES: usize = 50; + + for attempt in 0..MAX_RETRIES { + info!( + "RetryEvent: Trying to read index {:?}, attempt {}/{} ({}ms)", + event_index, + attempt + 1, + MAX_RETRIES, + SLEEP_MS * attempt as u64 + ); + match RetryEvent::check_uri(event_index).await { + Ok(timeframe) => { + if timeframe.is_some() { + return (); + } + } + Err(e) => panic!("Error while getting index: {:?}", e), + }; + // Nap time + tokio::time::sleep(Duration::from_millis(SLEEP_MS)).await; + } + panic!("TIMEOUT: It takes to much time to read the RetryManager new index") +} From 71c9d896464ce604dfe79b19c584ab7b63ef7fd4 Mon Sep 17 00:00:00 2001 From: tipogi Date: Wed, 29 Jan 2025 20:01:14 +0100 Subject: [PATCH 25/42] review fixes II --- tests/watcher/bookmarks/retry_bookmark.rs | 5 +---- tests/watcher/utils/watcher.rs | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index d7af4e1c..e03ce478 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::{try_until_write, WatcherTest}; +use crate::watcher::utils::watcher::WatcherTest; use anyhow::Result; use pubky_app_specs::{traits::HashId, PubkyAppBookmark, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -46,8 +46,6 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); - try_until_write(&put_index_key).await; - let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -69,7 +67,6 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { // DEL bookmark test.del(&bookmark_url).await?; - //tokio::time::sleep(Duration::from_millis(500)).await; let del_index_key = format!( "{}:{}", diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index 2778b75d..eebd1589 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -4,7 +4,7 @@ use std::time::Duration; use super::dht::TestnetDHTNetwork; use anyhow::{anyhow, Result}; use chrono::Utc; -use log::{debug, info}; +use log::debug; use pubky_app_specs::{ traits::TimestampId, PubkyAppFile, PubkyAppFollow, PubkyAppPost, PubkyAppUser, }; @@ -274,12 +274,12 @@ pub async fn retrieve_and_handle_event_line(event_line: &str) -> Result<(), DynE /// Attempts to read an event index with retries before timing out /// # Arguments /// * `event_index` - A string slice representing the index to check -pub async fn try_until_write(event_index: &str) { - const SLEEP_MS: u64 = 20; +pub async fn _assert_eventually_exists(event_index: &str) { + const SLEEP_MS: u64 = 3; const MAX_RETRIES: usize = 50; for attempt in 0..MAX_RETRIES { - info!( + debug!( "RetryEvent: Trying to read index {:?}, attempt {}/{} ({}ms)", event_index, attempt + 1, From bfdc0ce73ca3fa0ebe3c58be137154684ffc225c Mon Sep 17 00:00:00 2001 From: tipogi Date: Thu, 30 Jan 2025 07:43:05 +0100 Subject: [PATCH 26/42] minor changes II --- benches/watcher.rs | 6 +++--- examples/from_file.rs | 2 +- src/events/processor.rs | 13 ++++++++----- src/events/retry/manager.rs | 15 ++++++++------- src/watcher.rs | 2 +- tests/watcher/bookmarks/retry_bookmark.rs | 6 +++++- tests/watcher/follows/retry_follow.rs | 5 ++++- tests/watcher/mutes/retry_mute.rs | 6 +++++- tests/watcher/posts/retry_all.rs | 6 +++++- tests/watcher/posts/retry_post.rs | 6 +++++- tests/watcher/posts/retry_reply.rs | 6 +++++- tests/watcher/posts/retry_repost.rs | 6 +++++- tests/watcher/tags/retry_post_tag.rs | 6 +++++- tests/watcher/tags/retry_user_tag.rs | 6 +++++- tests/watcher/utils/watcher.rs | 4 ++-- 15 files changed, 67 insertions(+), 28 deletions(-) diff --git a/benches/watcher.rs b/benches/watcher.rs index 02843441..08a477b9 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -5,7 +5,7 @@ use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::{ - events::retry::manager::{RetryManager, SenderChannel}, + events::retry::manager::{RetryManager, RetryManagerSenderChannel}, EventProcessor, }; use setup::run_setup; @@ -19,7 +19,7 @@ mod setup; /// 2. Sign up the user /// 3. Upload a profile.json /// 4. Delete the profile.json -async fn create_homeserver_with_events() -> (Testnet, String, SenderChannel) { +async fn create_homeserver_with_events() -> (Testnet, String, RetryManagerSenderChannel) { // Create the test environment let testnet = Testnet::new(3).unwrap(); let homeserver = Homeserver::start_test(&testnet).await.unwrap(); @@ -34,7 +34,7 @@ async fn create_homeserver_with_events() -> (Testnet, String, SenderChannel) { let (receiver_channel, sender_channel) = RetryManager::init_channels(); // Create new asynchronous task to control the failed events - RetryManager::process_messages(&receiver_channel).await; + RetryManager::process_messages(receiver_channel).await; // Prepare the sender channel to send the messages to the retry manager let sender_clone = Arc::clone(&sender_channel); diff --git a/examples/from_file.rs b/examples/from_file.rs index 0bfab18f..f82e9c59 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -20,7 +20,7 @@ async fn main() -> Result<(), DynError> { let (receiver_channel, sender_channel) = RetryManager::init_channels(); // Create new asynchronous task to control the failed events - RetryManager::process_messages(&receiver_channel).await; + RetryManager::process_messages(receiver_channel).await; // Prepare the sender channel to send the messages to the retry manager let sender_clone = Arc::clone(&sender_channel); diff --git a/src/events/processor.rs b/src/events/processor.rs index 0d49b2ac..2d90de2c 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,5 +1,5 @@ use super::error::EventProcessorError; -use super::retry::manager::{RetryQueueMessage, SenderChannel}; +use super::retry::manager::{RetryManagerSenderChannel, RetryQueueMessage}; use super::Event; use crate::events::retry::event::RetryEvent; use crate::types::DynError; @@ -11,13 +11,13 @@ use log::{debug, error, info}; pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, - pub retry_manager_sender_channel: SenderChannel, + pub retry_manager_sender_channel: RetryManagerSenderChannel, } impl EventProcessor { pub async fn from_config( config: &Config, - retry_manager_sender_channel: SenderChannel, + retry_manager_sender_channel: RetryManagerSenderChannel, ) -> Result { let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; @@ -45,8 +45,11 @@ impl EventProcessor { /// /// # Parameters /// - `homeserver_id`: A `String` representing the URL of the homeserver to be used in the test environment. - /// - `tx`: A `SenderChannel` used to handle outgoing messages or events. - pub async fn test(homeserver_id: String, retry_manager_sender_channel: SenderChannel) -> Self { + /// - `tx`: A `RetryManagerSenderChannel` used to handle outgoing messages or events. + pub async fn test( + homeserver_id: String, + retry_manager_sender_channel: RetryManagerSenderChannel, + ) -> Self { let id = PubkyId(homeserver_id.to_string()); let homeserver = Homeserver::new(id).await.unwrap(); Self { diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index 9859b695..7c558577 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -22,8 +22,8 @@ pub enum RetryQueueMessage { ProcessEvent(String, RetryEvent), } -pub type SenderChannel = Arc>>; -type ReceiverChannel = Arc>>; +pub type RetryManagerSenderChannel = Arc>>; +//type RetryManagerReceiverChannel = Receiver; /// Manages the retry queue for processing failed events. /// @@ -39,10 +39,10 @@ pub struct RetryManager {} /// Initializes a new `RetryManager` with a message-passing channel impl RetryManager { - pub fn init_channels() -> (ReceiverChannel, SenderChannel) { + pub fn init_channels() -> (Receiver, RetryManagerSenderChannel) { let (tx, rx) = mpsc::channel(CHANNEL_BUFFER); ( - Arc::new(Mutex::new(rx)), // Receiver channel + rx, // Receiver channel Arc::new(Mutex::new(tx)), // Sender channel ) } @@ -52,10 +52,11 @@ impl RetryManager { /// based on their type /// The loop runs continuously until the channel is closed, ensuring that all messages /// are processed appropriately - pub async fn process_messages(receiver_channel: &ReceiverChannel) { - let receiver_channel = Arc::clone(receiver_channel); + /// + /// # Arguments + /// * `rx` - The receiver channel for retry messages + pub async fn process_messages(mut rx: Receiver) { tokio::spawn(async move { - let mut rx = receiver_channel.lock().await; // Listen all the messages in the channel while let Some(message) = rx.recv().await { match message { diff --git a/src/watcher.rs b/src/watcher.rs index 769803f7..e1bda107 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -20,7 +20,7 @@ async fn main() -> Result<(), Box> { let (receiver_channel, sender_channel) = RetryManager::init_channels(); // Create new asynchronous task to control the failed events - RetryManager::process_messages(&receiver_channel).await; + RetryManager::process_messages(receiver_channel).await; // Prepare the sender channel to send the messages to the retry manager let sender_clone = Arc::clone(&sender_channel); diff --git a/tests/watcher/bookmarks/retry_bookmark.rs b/tests/watcher/bookmarks/retry_bookmark.rs index e03ce478..f4a88371 100644 --- a/tests/watcher/bookmarks/retry_bookmark.rs +++ b/tests/watcher/bookmarks/retry_bookmark.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::{traits::HashId, PubkyAppBookmark, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -46,6 +46,8 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); + assert_eventually_exists(&put_index_key).await; + let timestamp = RetryEvent::check_uri(&put_index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -74,6 +76,8 @@ async fn test_homeserver_bookmark_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&bookmark_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index 057d357b..cdbd6e8b 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::PubkyAppUser; use pubky_common::crypto::Keypair; @@ -34,6 +34,7 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { EventType::Put, RetryEvent::generate_index_key(&follow_url).unwrap() ); + assert_eventually_exists(&index_key).await; let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -63,6 +64,8 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&follow_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index 661e02db..ef3062ac 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::PubkyAppUser; use pubky_common::crypto::Keypair; @@ -36,6 +36,8 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&mute_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -64,6 +66,8 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&mute_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/posts/retry_all.rs b/tests/watcher/posts/retry_all.rs index a639ec47..485e7e34 100644 --- a/tests/watcher/posts/retry_all.rs +++ b/tests/watcher/posts/retry_all.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -49,6 +49,8 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_reply_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -82,6 +84,8 @@ async fn test_homeserver_post_with_reply_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_reply_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/posts/retry_post.rs b/tests/watcher/posts/retry_post.rs index aaaa61e8..72c544ab 100644 --- a/tests/watcher/posts/retry_post.rs +++ b/tests/watcher/posts/retry_post.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind}; use pubky_common::crypto::Keypair; @@ -34,6 +34,8 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&post_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -65,6 +67,8 @@ async fn test_homeserver_post_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&post_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/posts/retry_reply.rs b/tests/watcher/posts/retry_reply.rs index 34427431..6dd6e7a9 100644 --- a/tests/watcher/posts/retry_reply.rs +++ b/tests/watcher/posts/retry_reply.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostKind, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -44,6 +44,8 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&reply_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -73,6 +75,8 @@ async fn test_homeserver_post_reply_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&reply_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/posts/retry_repost.rs b/tests/watcher/posts/retry_repost.rs index 17214734..4fdddb9c 100644 --- a/tests/watcher/posts/retry_repost.rs +++ b/tests/watcher/posts/retry_repost.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use pubky_app_specs::{PubkyAppPost, PubkyAppPostEmbed, PubkyAppPostKind, PubkyAppUser}; use pubky_common::crypto::Keypair; @@ -47,6 +47,8 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -76,6 +78,8 @@ async fn test_homeserver_post_repost_cannot_index() -> Result<()> { RetryEvent::generate_index_key(&repost_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/tags/retry_post_tag.rs b/tests/watcher/tags/retry_post_tag.rs index f563db12..bad37cc1 100644 --- a/tests/watcher/tags/retry_post_tag.rs +++ b/tests/watcher/tags/retry_post_tag.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use chrono::Utc; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; @@ -60,6 +60,8 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -90,6 +92,8 @@ async fn test_homeserver_post_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/tags/retry_user_tag.rs b/tests/watcher/tags/retry_user_tag.rs index 368447b9..4a61910f 100644 --- a/tests/watcher/tags/retry_user_tag.rs +++ b/tests/watcher/tags/retry_user_tag.rs @@ -1,4 +1,4 @@ -use crate::watcher::utils::watcher::WatcherTest; +use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; use chrono::Utc; use pubky_app_specs::{traits::HashId, PubkyAppTag, PubkyAppUser}; @@ -51,6 +51,8 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); + assert_eventually_exists(&index_key).await; + let timestamp = RetryEvent::check_uri(&index_key).await.unwrap(); assert!(timestamp.is_some()); @@ -80,6 +82,8 @@ async fn test_homeserver_user_tag_event_to_queue() -> Result<()> { RetryEvent::generate_index_key(&tag_url).unwrap() ); + assert_eventually_exists(&del_index_key).await; + let timestamp = RetryEvent::check_uri(&del_index_key).await.unwrap(); assert!(timestamp.is_some()); diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index eebd1589..02760840 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -53,7 +53,7 @@ impl WatcherTest { let (receiver_channel, sender_channel) = RetryManager::init_channels(); // Create new asynchronous task to control the failed events - RetryManager::process_messages(&receiver_channel).await; + RetryManager::process_messages(receiver_channel).await; // Prepare the sender channel to send the messages to the retry manager let sender_clone = Arc::clone(&sender_channel); @@ -274,7 +274,7 @@ pub async fn retrieve_and_handle_event_line(event_line: &str) -> Result<(), DynE /// Attempts to read an event index with retries before timing out /// # Arguments /// * `event_index` - A string slice representing the index to check -pub async fn _assert_eventually_exists(event_index: &str) { +pub async fn assert_eventually_exists(event_index: &str) { const SLEEP_MS: u64 = 3; const MAX_RETRIES: usize = 50; From 4888a037713926e973676cb3552465e3369f565f Mon Sep 17 00:00:00 2001 From: tipogi Date: Fri, 31 Jan 2025 12:04:13 +0100 Subject: [PATCH 27/42] improve channel stability between EventProcessor and RetryManager --- benches/watcher.rs | 18 +-- examples/from_file.rs | 15 +-- src/events/processor.rs | 227 +++++++++++++++++++++++---------- src/events/retry/manager.rs | 91 +++++++++---- src/watcher.rs | 13 +- tests/watcher/utils/watcher.rs | 13 +- 6 files changed, 244 insertions(+), 133 deletions(-) diff --git a/benches/watcher.rs b/benches/watcher.rs index 08a477b9..39f7f6bb 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -5,11 +5,11 @@ use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::{ - events::retry::manager::{RetryManager, RetryManagerSenderChannel}, + events::retry::manager::{RetryManager, WeakRetryManagerSenderChannel}, EventProcessor, }; use setup::run_setup; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use tokio::runtime::Runtime; mod setup; @@ -19,7 +19,7 @@ mod setup; /// 2. Sign up the user /// 3. Upload a profile.json /// 4. Delete the profile.json -async fn create_homeserver_with_events() -> (Testnet, String, RetryManagerSenderChannel) { +async fn create_homeserver_with_events() -> (Testnet, String, WeakRetryManagerSenderChannel) { // Create the test environment let testnet = Testnet::new(3).unwrap(); let homeserver = Homeserver::start_test(&testnet).await.unwrap(); @@ -30,14 +30,8 @@ async fn create_homeserver_with_events() -> (Testnet, String, RetryManagerSender let keypair = Keypair::random(); let user_id = keypair.public_key().to_z32(); - // Initializes a retry manager and ensures robustness by managing retries asynchronously - let (receiver_channel, sender_channel) = RetryManager::init_channels(); - - // Create new asynchronous task to control the failed events - RetryManager::process_messages(receiver_channel).await; - - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = Arc::clone(&sender_channel); + // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager + let sender_channel = RetryManager::clone_sender_channel(); // Create and delete a user profile (as per your requirement) client @@ -70,7 +64,7 @@ async fn create_homeserver_with_events() -> (Testnet, String, RetryManagerSender // Delete the user profile client.delete(url.as_str()).send().await.unwrap(); - (testnet, homeserver_url, sender_clone) + (testnet, homeserver_url, sender_channel) } fn bench_create_delete_user(c: &mut Criterion) { diff --git a/examples/from_file.rs b/examples/from_file.rs index f82e9c59..91f1f2c2 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -4,7 +4,6 @@ use pubky_nexus::{setup, types::DynError, Config, EventProcessor}; use std::fs::File; use std::io::{self, BufRead}; use std::path::Path; -use std::sync::Arc; // Create that file and add the file with that format // PUT homeserver_uri @@ -15,17 +14,11 @@ const FILE_PATH: &str = "examples/events.txt"; async fn main() -> Result<(), DynError> { let config = Config::from_env(); setup(&config).await; + + // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager + let sender_channel = RetryManager::clone_sender_channel(); - // Initializes a retry manager and ensures robustness by managing retries asynchronously - let (receiver_channel, sender_channel) = RetryManager::init_channels(); - - // Create new asynchronous task to control the failed events - RetryManager::process_messages(receiver_channel).await; - - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = Arc::clone(&sender_channel); - - let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; + let mut event_processor = EventProcessor::from_config(&config, sender_channel).await?; let events = read_events_from_file().unwrap(); event_processor.process_event_lines(events).await?; diff --git a/src/events/processor.rs b/src/events/processor.rs index 2d90de2c..c060ca93 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,39 +1,34 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + use super::error::EventProcessorError; -use super::retry::manager::{RetryManagerSenderChannel, RetryQueueMessage}; +use super::retry::manager::{RetryQueueMessage, WeakRetryManagerSenderChannel}; use super::Event; use crate::events::retry::event::RetryEvent; +use crate::events::retry::manager::RetryManager; use crate::types::DynError; use crate::types::PubkyId; use crate::PubkyConnector; use crate::{models::homeserver::Homeserver, Config}; -use log::{debug, error, info}; +use log::{debug, error, info, warn}; +use tokio::{sync::mpsc, time::Duration}; + +// REVIEW: This could be env variables and can be part of EventProcessor +const MAX_ATTEMPTS: i32 = 5; +const MESSAGE_SEND_RETRY: u64 = 10; +const RETRY_THRESHOLD: usize = 20; pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, - pub retry_manager_sender_channel: RetryManagerSenderChannel, + pub retry_manager_sender_channel: WeakRetryManagerSenderChannel, + consecutive_message_send_failures: Arc, } impl EventProcessor { - pub async fn from_config( - config: &Config, - retry_manager_sender_channel: RetryManagerSenderChannel, - ) -> Result { - let homeserver = Homeserver::from_config(config).await?; - let limit = config.events_limit; - - info!( - "Initialized Event Processor for homeserver: {:?}", - homeserver - ); - - Ok(Self { - homeserver, - limit, - retry_manager_sender_channel, - }) + pub fn set_sender_channel(&mut self, channel: WeakRetryManagerSenderChannel) { + self.retry_manager_sender_channel = channel; } - /// Creates a new `EventProcessor` instance for testing purposes. /// /// This function initializes an `EventProcessor` configured with: @@ -48,7 +43,7 @@ impl EventProcessor { /// - `tx`: A `RetryManagerSenderChannel` used to handle outgoing messages or events. pub async fn test( homeserver_id: String, - retry_manager_sender_channel: RetryManagerSenderChannel, + retry_manager_sender_channel: WeakRetryManagerSenderChannel, ) -> Self { let id = PubkyId(homeserver_id.to_string()); let homeserver = Homeserver::new(id).await.unwrap(); @@ -56,9 +51,30 @@ impl EventProcessor { homeserver, limit: 1000, retry_manager_sender_channel, + consecutive_message_send_failures: Arc::new(AtomicUsize::new(0)), } } + pub async fn from_config( + config: &Config, + retry_manager_sender_channel: WeakRetryManagerSenderChannel, + ) -> Result { + let homeserver = Homeserver::from_config(config).await?; + let limit = config.events_limit; + + info!( + "Initialized Event Processor for homeserver: {:?}", + homeserver + ); + + Ok(Self { + homeserver, + limit, + retry_manager_sender_channel, + consecutive_message_send_failures: Arc::new(AtomicUsize::new(0)), + }) + } + pub async fn run(&mut self) -> Result<(), DynError> { let lines = { self.poll_events().await.unwrap_or_default() }; if let Some(lines) = lines { @@ -135,57 +151,132 @@ impl EventProcessor { Ok(()) } - // Generic retry on event handler - async fn handle_event(&self, event: Event) -> Result<(), DynError> { - match event.clone().handle().await { - Ok(_) => Ok(()), - Err(e) => { - let retry_event = match e.downcast_ref::() { - Some(EventProcessorError::InvalidEventLine { message }) => { - error!("{}", message); - return Ok(()); - } - Some(event_processor_error) => RetryEvent::new(event_processor_error.clone()), - // Others errors must be logged at least for now - None => { - error!("Unhandled error type for URI: {}, {:?}", event.uri, e); - return Ok(()); - } - }; + /// Processes an event and track the fail event it if necessary + /// # Parameters: + /// - `event`: The event to be processed + async fn handle_event(&mut self, event: Event) -> Result<(), DynError> { + let mut attempts = 0; + let mut delay = Duration::from_millis(MESSAGE_SEND_RETRY); + + if let Err(e) = event.clone().handle().await { + if let Some((index_key, retry_event)) = extract_retry_event_info(&event, e) { + let queue_message = RetryQueueMessage::ProcessEvent(index_key.clone(), retry_event); + + while attempts < MAX_ATTEMPTS { + let mut restart = false; + if let Some(sender_arc) = self.retry_manager_sender_channel.upgrade() { + let mut sender_guard = sender_arc.lock().await; - // Generate a compress index to save in the cache - let index = match RetryEvent::generate_index_key(&event.uri) { - Some(retry_index) => retry_index, - None => { - // Unlikely to be reached, as it would typically fail during the validation process - return Ok(()); + if let Some(sender) = sender_guard.as_mut() { + match sender.try_send(queue_message.clone()) { + Err(mpsc::error::TrySendError::Full(_)) => { + warn!( + "Retry channel is full. Retrying in {:?}... (attempt {}/{})", + delay, attempts + 1, MAX_ATTEMPTS + ); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + warn!("Retry channel receiver is unavailable! Restarting RetryManager..."); + restart = true; + } + _ => { + // Message sent successfully, reset failure counter + self.consecutive_message_send_failures + .store(0, Ordering::Relaxed); + return Ok(()); + } + } + } else { + warn!("Retry sender channel has been dropped. Retrying in {:?}... (attempt {}/{})", + delay, attempts + 1, MAX_ATTEMPTS + ); + } + } else { + warn!("Sender is invalid/dropped. Retrying event in next iteration..."); } - }; - let index_key = format!("{}:{}", event.event_type, index); - - // Send event to the retry manager - let sender_channel = self.retry_manager_sender_channel.lock().await; - match sender_channel - .send(RetryQueueMessage::ProcessEvent(index_key, retry_event)) - .await - { - Ok(_) => { - // TODO: Investigate non-blocking alternatives - // The current use of `tokio::time::sleep` (in the watcher tests) is intended to handle a situation where tasks in other threads - // are not being tracked. This could potentially lead to issues with writing `RetryEvents` in certain cases. - // Considerations: - // - Determine if this delay is genuinely necessary. Testing may reveal that the `RetryManager` thread - // handles retries adequately while the watcher remains active, making this timer redundant. - // - If the delay is required, explore a more efficient solution that does not block the thread - // and ensures proper handling of tasks in other threads. - // - // For now, the sleep is deactivated - //tokio::time::sleep(Duration::from_millis(500)).await; + + if restart { + self.update_sender_channel().await; + break; } - Err(e) => error!("Failed to send message to RetryManager: {:?}", e), + + tokio::time::sleep(delay).await; + // Apply exponential backoff before the next retry attempt + delay *= 2; + attempts += 1; } - Ok(()) + + error!( + "Message passing failed: Unable to send event {:?} after {} attempts. Dropping event...", + index_key, MAX_ATTEMPTS + ); + + // Increase failure counter to track consecutive failures + self.consecutive_message_send_failures + .fetch_add(1, Ordering::Relaxed); + self.evaluate_message_send_failures().await; } } + Ok(()) + } + + /// Checks if the consecutive message send failure count has reached RETRY_THRESHOLD + /// and triggers a RetryManager restart if necessary + /// This prevents RetryManager from restarting too frequently + async fn evaluate_message_send_failures(&mut self) { + // Before reset the retry manager, reset the consecutive message send fail counter + let reset_needed = self + .consecutive_message_send_failures + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + if count >= RETRY_THRESHOLD { + Some(0) // Reset counter + } else { + None + } + }) + .is_ok(); + + if reset_needed { + warn!( + "Message send failure threshold reached ({} consecutive failures). Restarting RetryManager...", + RETRY_THRESHOLD + ); + self.update_sender_channel().await; + } + } + + /// Restarts the RetryManager and updates the sender channel reference + async fn update_sender_channel(&mut self) { + RetryManager::restart().await; + self.set_sender_channel(RetryManager::clone_sender_channel()); } } + +/// Extracts retry-related information from an event and its associated error +/// +/// # Parameters +/// - `event`: Reference to the event for which retry information is being extracted +/// - `error`: Determines whether the event is eligible for a retry or should be discarded +fn extract_retry_event_info(event: &Event, error: DynError) -> Option<(String, RetryEvent)> { + let retry_event = match error.downcast_ref::() { + Some(EventProcessorError::InvalidEventLine { message }) => { + error!("{}", message); + return None; + } + Some(event_processor_error) => RetryEvent::new(event_processor_error.clone()), + // Others errors must be logged at least for now + None => { + error!("Unhandled error type for URI: {}, {:?}", event.uri, error); + return None; + } + }; + + // Generate a compress index to save in the cache + let index = match RetryEvent::generate_index_key(&event.uri) { + Some(retry_index) => retry_index, + None => { + return None; + } + }; + Some((format!("{}:{}", event.event_type, index), retry_event)) +} diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs index 7c558577..91e5587d 100644 --- a/src/events/retry/manager.rs +++ b/src/events/retry/manager.rs @@ -1,5 +1,6 @@ -use log::{debug, error}; -use std::sync::Arc; +use log::{debug, error, info}; +use once_cell::sync::OnceCell; +use std::sync::{Arc, Weak}; use tokio::sync::mpsc; use tokio::sync::{ mpsc::{Receiver, Sender}, @@ -9,6 +10,7 @@ use tokio::sync::{ use super::event::RetryEvent; pub const CHANNEL_BUFFER: usize = 1024; +pub static RETRY_MANAGER: OnceCell = OnceCell::new(); /// Represents commands sent to the retry manager for handling events in the retry queue #[derive(Debug, Clone)] @@ -22,7 +24,7 @@ pub enum RetryQueueMessage { ProcessEvent(String, RetryEvent), } -pub type RetryManagerSenderChannel = Arc>>; +pub type WeakRetryManagerSenderChannel = Weak>>>; //type RetryManagerReceiverChannel = Receiver; /// Manages the retry queue for processing failed events. @@ -35,16 +37,63 @@ pub type RetryManagerSenderChannel = Arc>>; /// - Receives and processes messages related to event retries /// - Maintains a queue of events that need to be retried /// - Ensures failed events are reprocessed in an orderly manner -pub struct RetryManager {} +#[derive(Debug, Clone)] +pub struct RetryManager { + sender_channel: Arc>>>, +} /// Initializes a new `RetryManager` with a message-passing channel impl RetryManager { - pub fn init_channels() -> (Receiver, RetryManagerSenderChannel) { - let (tx, rx) = mpsc::channel(CHANNEL_BUFFER); - ( - rx, // Receiver channel - Arc::new(Mutex::new(tx)), // Sender channel - ) + pub fn initialise() -> &'static RetryManager { + RETRY_MANAGER.get_or_init(|| { + let (tx, rx) = mpsc::channel(CHANNEL_BUFFER); + + tokio::spawn(Self::process_messages(rx)); + + RetryManager { + sender_channel: Arc::new(Mutex::new(Some(tx))), + } + }) + } + + /// Returns a weak reference to the sender channel + /// + /// It clones the sender channel as a `Weak` reference instead of `Arc`, + /// preventing strong ownership cycles that could lead to memory leaks. By using `Weak`, + /// the sender channel can be automatically cleaned up when all strong references + /// (`Arc`) to it are dropped. This ensures that: + /// + /// - The sender channel is **not kept alive indefinitely** if the `RetryManager` is restarted + /// - It allows us to **check if the sender is still valid** before attempting to send messages + /// - Prevents **unnecessary memory usage** by ensuring unused channels are cleaned up + /// + /// # Returns: + /// - A `Weak` reference to the sender channel, which can be upgraded to `Arc` when needed + pub fn clone_sender_channel() -> Weak>>> { + Arc::downgrade(&Self::initialise().sender_channel) + } + + /// Replaces the existing sender channel with a new one and restarts the communication channel + pub async fn restart() { + let (new_sender_channel, new_receiver_channel) = mpsc::channel(CHANNEL_BUFFER); + + { + let mut sender_lock = Self::initialise() + .sender_channel + .lock() + .await; + *sender_lock = None; // Remove old sender + } + + tokio::spawn(RetryManager::process_messages(new_receiver_channel)); + + let mut sender_lock = Self::initialise() + .sender_channel + .lock() + .await; + *sender_lock = Some(new_sender_channel.clone()); + + info!("RetryManager restarted the channel communication"); } /// Executes the main event loop to process messages from the channel @@ -55,20 +104,18 @@ impl RetryManager { /// /// # Arguments /// * `rx` - The receiver channel for retry messages - pub async fn process_messages(mut rx: Receiver) { - tokio::spawn(async move { - // Listen all the messages in the channel - while let Some(message) = rx.recv().await { - match message { - RetryQueueMessage::ProcessEvent(index_key, retry_event) => { - error!("{}, {}", retry_event.error_type, index_key); - if let Err(err) = retry_event.put_to_index(index_key).await { - error!("Failed to put event to index: {}", err); - } + async fn process_messages(mut rx: Receiver) { + // Listen all the messages in the channel + while let Some(message) = rx.recv().await { + match message { + RetryQueueMessage::ProcessEvent(index_key, retry_event) => { + error!("{}, {}", retry_event.error_type, index_key); + if let Err(err) = retry_event.put_to_index(index_key).await { + error!("Failed to put event to index: {}", err); } - _ => debug!("New message received, not handled: {:?}", message), } + _ => debug!("New message received, not handled: {:?}", message), } - }); + } } } diff --git a/src/watcher.rs b/src/watcher.rs index e1bda107..73b556e1 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -3,7 +3,6 @@ use log::info; use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::PubkyConnector; use pubky_nexus::{setup, Config, EventProcessor}; -use std::sync::Arc; use tokio::time::{sleep, Duration}; /// Watches over a homeserver `/events` and writes into the Nexus databases @@ -16,17 +15,11 @@ async fn main() -> Result<(), Box> { // Initializes the PubkyConnector with the configuration PubkyConnector::initialise(&config, None).await?; - // Initializes a retry manager and ensures robustness by managing retries asynchronously - let (receiver_channel, sender_channel) = RetryManager::init_channels(); - - // Create new asynchronous task to control the failed events - RetryManager::process_messages(receiver_channel).await; - - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = Arc::clone(&sender_channel); + // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager + let sender_channel = RetryManager::clone_sender_channel(); // Create and configure the event processor - let mut event_processor = EventProcessor::from_config(&config, sender_clone).await?; + let mut event_processor = EventProcessor::from_config(&config, sender_channel).await?; loop { info!("Fetching events..."); diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index 02760840..09b04ee5 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -1,4 +1,3 @@ -use std::sync::Arc; use std::time::Duration; use super::dht::TestnetDHTNetwork; @@ -49,21 +48,15 @@ impl WatcherTest { let homeserver = Homeserver::start_test(&testnet).await?; let homeserver_id = homeserver.public_key().to_string(); - // Initializes a retry manager and ensures robustness by managing retries asynchronously - let (receiver_channel, sender_channel) = RetryManager::init_channels(); - - // Create new asynchronous task to control the failed events - RetryManager::process_messages(receiver_channel).await; - - // Prepare the sender channel to send the messages to the retry manager - let sender_clone = Arc::clone(&sender_channel); + // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager + let sender_channel = RetryManager::clone_sender_channel(); match PubkyConnector::initialise(&config, Some(&testnet)).await { Ok(_) => debug!("WatcherTest: PubkyConnector initialised"), Err(e) => debug!("WatcherTest: {}", e), } - let event_processor = EventProcessor::test(homeserver_id, sender_clone).await; + let event_processor = EventProcessor::test(homeserver_id, sender_channel).await; Ok(Self { config, From e2faab07c789af0708148b3783c71e50c18d1f9c Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Sun, 2 Feb 2025 20:04:26 -0400 Subject: [PATCH 28/42] Init use specs parser --- Cargo.lock | 82 ++++++------ Cargo.toml | 2 +- src/db/graph/queries/put.rs | 7 +- src/events/handlers/bookmark.rs | 13 +- src/events/handlers/post.rs | 25 +++- src/events/handlers/tag.rs | 17 ++- src/events/mod.rs | 224 +++++++++----------------------- src/events/uri.rs | 122 ----------------- 8 files changed, 145 insertions(+), 347 deletions(-) delete mode 100644 src/events/uri.rs diff --git a/Cargo.lock b/Cargo.lock index 6a6ff249..de0ddf3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,13 +154,13 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -328,7 +328,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -486,9 +486,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.10" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" dependencies = [ "shlex", ] @@ -609,7 +609,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -899,7 +899,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -972,7 +972,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1015,7 +1015,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1044,9 +1044,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" [[package]] name = "ed25519" @@ -1283,7 +1283,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1877,7 +1877,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2223,7 +2223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a0d57c55d2d1dc62a2b1d16a0a1079eb78d67c36bdf468d582ab4482ec7002" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2328,9 +2328,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.69" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e534d133a060a3c19daec1eb3e98ec6f4685978834f2dbadfe2ec215bab64e" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags", "cfg-if", @@ -2349,7 +2349,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2360,9 +2360,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -2487,7 +2487,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2516,7 +2516,7 @@ checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2730,7 +2730,7 @@ dependencies = [ [[package]] name = "pubky-app-specs" version = "0.3.0" -source = "git+https://github.com/pubky/pubky-app-specs#9278a996d0804cd5b15d58dae9fc606ea0f4977c" +source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#3ce87b2f8fe92a3985faebbdde49686e64296dcf" dependencies = [ "base32", "blake3", @@ -3204,7 +3204,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.96", + "syn 2.0.98", "walkdir", ] @@ -3435,7 +3435,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3647,9 +3647,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -3682,7 +3682,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3746,7 +3746,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3757,7 +3757,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3862,7 +3862,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3904,7 +3904,7 @@ checksum = "1fe49a94e3a984b0d0ab97343dc3dcd52baae1ee13f005bfad39faea47d051dc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4075,7 +4075,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4210,7 +4210,7 @@ source = "git+https://github.com/juhaku/utoipa?rev=d522f744259dc4fde5f45d187983f dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4304,7 +4304,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-shared", ] @@ -4339,7 +4339,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4584,7 +4584,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -4606,7 +4606,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4626,7 +4626,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -4655,7 +4655,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4687,4 +4687,4 @@ dependencies = [ "log", "once_cell", "simd-adler32", -] \ No newline at end of file +] diff --git a/Cargo.toml b/Cargo.toml index b8700d12..6392cb52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ bytes = "1.9.0" # Enforce this version as Pubky and Axum conflict v1.7.1 vs v1.9 pkarr = { git = "https://github.com/pubky/pkarr", branch = "v3-rc1", package = "pkarr" } mainline = { git = "https://github.com/pubky/mainline", branch = "v5-rc1", default-features = false } pubky = { git = "https://github.com/pubky/pubky", branch = "v0.4.0-rc1" } -pubky-app-specs = { git = "https://github.com/pubky/pubky-app-specs", features = [ +pubky-app-specs = { git = "https://github.com/pubky/pubky-app-specs", branch = "feat/uri-parser", features = [ "openapi", ] } tokio = { version = "1.43.0", features = ["full"] } diff --git a/src/db/graph/queries/put.rs b/src/db/graph/queries/put.rs index c8abd309..9beb16ef 100644 --- a/src/db/graph/queries/put.rs +++ b/src/db/graph/queries/put.rs @@ -1,8 +1,8 @@ -use crate::events::uri::ParsedUri; use crate::models::post::PostRelationships; use crate::models::{file::FileDetails, post::PostDetails, user::UserDetails}; use crate::types::DynError; use neo4rs::{query, Query}; +use pubky_app_specs::{ParsedUri, Resource}; // Create a user node pub fn create_user(user: &UserDetails) -> Result { @@ -110,7 +110,10 @@ fn add_relationship_params( if let Some(uri) = uri { let parsed_uri = ParsedUri::try_from(uri.as_str())?; let parent_author_id = parsed_uri.user_id; - let parent_post_id = parsed_uri.post_id.ok_or("Missing post ID")?; + let parent_post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Reposted uri is not a Post resource".into()), + }; return Ok(cypher_query .param(author_param, parent_author_id.as_str()) diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 8354fea4..42deae16 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -1,14 +1,13 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; -use crate::events::uri::ParsedUri; use crate::models::post::Bookmark; use crate::models::user::UserCounts; use crate::types::DynError; use chrono::Utc; use log::debug; use pubky_app_specs::traits::Validatable; -use pubky_app_specs::{PubkyAppBookmark, PubkyId}; +use pubky_app_specs::{ParsedUri, PubkyAppBookmark, PubkyId, Resource}; //TODO: only /posts/ are bookmarkable as of now. pub async fn put(user_id: PubkyId, bookmark_id: String, blob: &[u8]) -> Result<(), DynError> { @@ -27,10 +26,12 @@ pub async fn sync_put( ) -> Result<(), DynError> { // Parse the URI to extract author_id and post_id using the updated parse_post_uri let parsed_uri = ParsedUri::try_from(bookmark.uri.as_str())?; - let (author_id, post_id) = ( - parsed_uri.user_id, - parsed_uri.post_id.ok_or("Bookmarked URI missing post_id")?, - ); + let author_id = parsed_uri.user_id; + let post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Bookmarked uri is not a Post resource".into()), + }; + // Save new bookmark relationship to the graph, only if the bookmarked user exists let indexed_at = Utc::now().timestamp_millis(); let existed = diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index 625bad44..d33c7488 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -2,7 +2,6 @@ use crate::db::graph::exec::{exec_single_row, execute_graph_operation, Operation use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; use crate::events::retry::event::RetryEvent; -use crate::events::uri::ParsedUri; use crate::models::notification::{Notification, PostChangedSource, PostChangedType}; use crate::models::post::{ PostCounts, PostDetails, PostRelationships, PostStream, POST_TOTAL_ENGAGEMENT_KEY_PARTS, @@ -12,7 +11,9 @@ use crate::queries::get::post_is_safe_to_delete; use crate::types::DynError; use crate::{queries, RedisOps, ScoreAction}; use log::debug; -use pubky_app_specs::{traits::Validatable, PubkyAppPost, PubkyAppPostKind, PubkyId}; +use pubky_app_specs::{ + traits::Validatable, ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource, +}; use super::utils::post_relationships_is_reply; @@ -108,7 +109,10 @@ pub async fn sync_put( let parsed_uri = ParsedUri::try_from(replied_uri.as_str())?; let parent_author_id = parsed_uri.user_id; - let parent_post_id = parsed_uri.post_id.ok_or("Missing post ID")?; + let parent_post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Reposted uri is not a Post resource".into()), + }; let parent_post_key_parts: &[&str; 2] = &[&parent_author_id, &parent_post_id]; @@ -149,7 +153,10 @@ pub async fn sync_put( let parsed_uri = ParsedUri::try_from(reposted_uri.as_str())?; let parent_author_id = parsed_uri.user_id; - let parent_post_id = parsed_uri.post_id.ok_or("Missing post ID")?; + let parent_post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Reposted uri is not a Post resource".into()), + }; let parent_post_key_parts: &[&str; 2] = &[&parent_author_id, &parent_post_id]; @@ -317,7 +324,10 @@ pub async fn sync_del(author_id: PubkyId, post_id: String) -> Result<(), DynErro // Decrement counts for resposted post if existed if let Some(reposted) = relationships.reposted { let parsed_uri = ParsedUri::try_from(reposted.as_str())?; - let parent_post_id = parsed_uri.post_id.ok_or("Missing post ID")?; + let parent_post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Reposted uri is not a Post resource".into()), + }; let parent_post_key_parts: &[&str] = &[&parsed_uri.user_id, &parent_post_id]; @@ -353,7 +363,10 @@ pub async fn sync_del(author_id: PubkyId, post_id: String) -> Result<(), DynErro if let Some(replied) = relationships.replied { let parsed_uri = ParsedUri::try_from(replied.as_str())?; let parent_user_id = parsed_uri.user_id; - let parent_post_id = parsed_uri.post_id.ok_or("Missing post ID")?; + let parent_post_id = match parsed_uri.resource { + Resource::Post(id) => id, + _ => return Err("Replied uri is not a Post resource".into()), + }; let parent_post_key_parts: [&str; 2] = [&parent_user_id, &parent_post_id]; reply_parent_post_key_wrapper = diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index f0761ed4..5d13868d 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -1,7 +1,6 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; -use crate::events::uri::ParsedUri; use crate::models::notification::Notification; use crate::models::post::{PostCounts, PostStream}; use crate::models::tag::post::TagPost; @@ -14,7 +13,8 @@ use crate::types::DynError; use crate::ScoreAction; use chrono::Utc; use log::debug; -use pubky_app_specs::{traits::Validatable, PubkyAppTag, PubkyId}; +use pubky_app_specs::Resource; +use pubky_app_specs::{traits::Validatable, ParsedUri, PubkyAppTag, PubkyId}; use super::utils::post_relationships_is_reply; @@ -28,9 +28,9 @@ pub async fn put(tagger_id: PubkyId, tag_id: String, blob: &[u8]) -> Result<(), let parsed_uri = ParsedUri::try_from(tag.uri.as_str())?; let indexed_at = Utc::now().timestamp_millis(); - match parsed_uri.post_id { + match parsed_uri.resource { // If post_id is in the tagged URI, we place tag to a post. - Some(post_id) => { + Resource::Post(post_id) => { put_sync_post( tagger_id, parsed_uri.user_id, @@ -43,7 +43,14 @@ pub async fn put(tagger_id: PubkyId, tag_id: String, blob: &[u8]) -> Result<(), .await } // If no post_id in the tagged URI, we place tag to a user. - None => put_sync_user(tagger_id, parsed_uri.user_id, tag_id, tag.label, indexed_at).await, + Resource::User => { + put_sync_user(tagger_id, parsed_uri.user_id, tag_id, tag.label, indexed_at).await + } + other => Err(format!( + "The tagged resource is not Post or User resource. Tagged resource: {:?}", + other + ) + .into()), } } diff --git a/src/events/mod.rs b/src/events/mod.rs index a1de604f..e41dafb4 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -1,48 +1,14 @@ use crate::{db::connectors::pubky::PubkyConnector, types::DynError}; use error::EventProcessorError; use log::debug; -use pubky_app_specs::PubkyId; -use reqwest; +use pubky_app_specs::{ParsedUri, Resource}; use serde::{Deserialize, Serialize}; use std::fmt; -use uri::ParsedUri; pub mod error; pub mod handlers; pub mod processor; pub mod retry; -pub mod uri; - -#[derive(Debug, Clone)] -enum ResourceType { - User { - user_id: PubkyId, - }, - Post { - author_id: PubkyId, - post_id: String, - }, - Follow { - follower_id: PubkyId, - followee_id: PubkyId, - }, - Mute { - user_id: PubkyId, - muted_id: PubkyId, - }, - Bookmark { - user_id: PubkyId, - bookmark_id: String, - }, - Tag { - user_id: PubkyId, - tag_id: String, - }, - File { - user_id: PubkyId, - file_id: String, - }, -} // Look for the end pattern after the start index, or use the end of the string if not found #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -63,14 +29,14 @@ impl fmt::Display for EventType { #[derive(Debug, Clone)] pub struct Event { - uri: String, - event_type: EventType, - resource_type: ResourceType, + pub uri: String, + pub event_type: EventType, + pub parsed_uri: ParsedUri, } impl Event { pub fn parse_event(line: &str) -> Result, DynError> { - debug!("New event: {}", line); + log::debug!("New event: {}", line); let parts: Vec<&str> = line.split(' ').collect(); if parts.len() != 2 { return Err(EventProcessorError::InvalidEventLine { @@ -82,87 +48,28 @@ impl Event { let event_type = match parts[0] { "PUT" => EventType::Put, "DEL" => EventType::Del, - _ => { + other => { return Err(EventProcessorError::InvalidEventLine { - message: format!("Unknown event type, {}", line), + message: format!("Unknown event type: {}", other), } .into()) } }; + // Validate and parse the URI using pubky-app-specs let uri = parts[1].to_string(); - let parsed_uri = ParsedUri::try_from(uri.as_str())?; - - //TODO: This conversion to a match statement that only uses IF conditions is silly. - // We could be patter matching the split test for "posts", "follows", etc maybe? - let resource_type = match uri { - _ if uri.ends_with("/pub/pubky.app/profile.json") => ResourceType::User { - user_id: parsed_uri.user_id, - }, - _ if uri.contains("/posts/") => ResourceType::Post { - author_id: parsed_uri.user_id, - post_id: parsed_uri - .post_id - .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing post_id, {}", line), - })?, - }, - _ if uri.contains("/follows/") => ResourceType::Follow { - follower_id: parsed_uri.user_id, - followee_id: parsed_uri - .follow_id - .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing followee_id, {}", line), - })?, - }, - _ if uri.contains("/mutes/") => ResourceType::Mute { - user_id: parsed_uri.user_id, - muted_id: parsed_uri - .muted_id - .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing muted_id, {}", line), - })?, - }, - _ if uri.contains("/bookmarks/") => ResourceType::Bookmark { - user_id: parsed_uri.user_id, - bookmark_id: parsed_uri.bookmark_id.ok_or( - EventProcessorError::InvalidEventLine { - message: format!("Missing bookmark_id, {}", line), - }, - )?, - }, - _ if uri.contains("/tags/") => ResourceType::Tag { - user_id: parsed_uri.user_id, - tag_id: parsed_uri - .tag_id - .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing tag_id, {}", line), - })?, - }, - _ if uri.contains("/files/") => ResourceType::File { - user_id: parsed_uri.user_id, - file_id: parsed_uri - .file_id - .ok_or(EventProcessorError::InvalidEventLine { - message: format!("Missing file_id, {}", line), - })?, - }, - _ if uri.contains("/blobs") => return Ok(None), - _ if uri.contains("/last_read") => return Ok(None), - _ if uri.contains("/settings") => return Ok(None), - _ if uri.contains("/feeds") => return Ok(None), - _ => { - return Err(EventProcessorError::InvalidEventLine { - message: format!("Unrecognized resource in URI, {}", line), + let parsed_uri = ParsedUri::try_from(uri.as_str()).map_err(|e| { + { + EventProcessorError::InvalidEventLine { + message: format!("Cannot parse event URI: {}", e), } - .into()) } - }; + })?; Ok(Some(Event { uri, event_type, - resource_type, + parsed_uri, })) } @@ -173,78 +80,67 @@ impl Event { } } - async fn handle_put_event(self) -> Result<(), DynError> { - debug!("Handling PUT event for {:?}", self.resource_type); - - // User PUT event's into the homeserver write new data. We fetch the data - // for every Resource Type - let url = reqwest::Url::parse(&self.uri)?; - let pubky_client = PubkyConnector::get_pubky_client()?; - - let response = match pubky_client.get(url).send().await { - Ok(response) => response, - Err(e) => { - return Err(EventProcessorError::PubkyClientError { - message: format!("{}", e), + /// Handles a PUT event by fetching the blob from the homeserver + /// and using the importer to convert it to a PubkyAppObject. + pub async fn handle_put_event(self) -> Result<(), DynError> { + log::debug!("Handling PUT event for URI: {}", self.uri); + + let response; + { + let pubky_client = PubkyConnector::get_pubky_client()?; + response = match pubky_client.get(&self.uri).send().await { + Ok(response) => response, + Err(e) => { + return Err(EventProcessorError::PubkyClientError { + message: format!("{}", e), + } + .into()) } - .into()) - } - }; + }; + } // drop the pubky_client lock let blob = response.bytes().await?; - match self.resource_type { - ResourceType::User { user_id } => handlers::user::put(user_id, &blob).await?, - ResourceType::Post { author_id, post_id } => { - handlers::post::put(author_id, post_id, &blob).await? + let user_id = self.parsed_uri.user_id; + match self.parsed_uri.resource { + Resource::User => handlers::user::put(user_id, &blob).await?, + Resource::Post(post_id) => handlers::post::put(user_id, post_id, &blob).await?, + Resource::Follow(followee_id) => { + handlers::follow::put(user_id, followee_id, &blob).await? } - ResourceType::Follow { - follower_id, - followee_id, - } => handlers::follow::put(follower_id, followee_id, &blob).await?, - ResourceType::Mute { user_id, muted_id } => { - handlers::mute::put(user_id, muted_id, &blob).await? + Resource::Mute(muted_id) => handlers::mute::put(user_id, muted_id, &blob).await?, + Resource::Bookmark(bookmark_id) => { + handlers::bookmark::put(user_id, bookmark_id, &blob).await? } - ResourceType::Bookmark { - user_id, - bookmark_id, - } => handlers::bookmark::put(user_id, bookmark_id, &blob).await?, - ResourceType::Tag { user_id, tag_id } => { - handlers::tag::put(user_id, tag_id, &blob).await? - } - ResourceType::File { user_id, file_id } => { + Resource::Tag(tag_id) => handlers::tag::put(user_id, tag_id, &blob).await?, + Resource::File(file_id) => { handlers::file::put(self.uri, user_id, file_id, &blob).await? } + other => { + log::debug!("Event type not handled, Resource: {:?}", other); + } } - Ok(()) } - async fn handle_del_event(self) -> Result<(), DynError> { - debug!("Handling DEL event for {:?}", self.resource_type); - - match self.resource_type { - ResourceType::User { user_id } => handlers::user::del(user_id).await?, - ResourceType::Post { author_id, post_id } => { - handlers::post::del(author_id, post_id).await? + pub async fn handle_del_event(self) -> Result<(), DynError> { + debug!("Handling DEL event for URI: {}", self.uri); + + let user_id = self.parsed_uri.user_id; + match self.parsed_uri.resource { + Resource::User => handlers::user::del(user_id).await?, + Resource::Post(post_id) => handlers::post::del(user_id, post_id).await?, + Resource::Follow(followee_id) => handlers::follow::del(user_id, followee_id).await?, + Resource::Mute(muted_id) => handlers::mute::del(user_id, muted_id).await?, + Resource::Bookmark(bookmark_id) => { + handlers::bookmark::del(user_id, bookmark_id).await? } - ResourceType::Follow { - follower_id, - followee_id, - } => handlers::follow::del(follower_id, followee_id).await?, - ResourceType::Mute { user_id, muted_id } => { - handlers::mute::del(user_id, muted_id).await? - } - ResourceType::Bookmark { - user_id, - bookmark_id, - } => handlers::bookmark::del(user_id, bookmark_id).await?, - ResourceType::Tag { user_id, tag_id } => handlers::tag::del(user_id, tag_id).await?, - ResourceType::File { user_id, file_id } => { - handlers::file::del(&user_id, file_id).await? + Resource::Tag(tag_id) => handlers::tag::del(user_id, tag_id).await?, + Resource::File(file_id) => handlers::file::del(&user_id, file_id).await?, + other => { + debug!("DEL event type not handled for resource: {:?}", other); } } - Ok(()) } } diff --git a/src/events/uri.rs b/src/events/uri.rs deleted file mode 100644 index f127194c..00000000 --- a/src/events/uri.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::types::DynError; -use pubky_app_specs::PubkyId; -use std::convert::TryFrom; - -use super::error::EventProcessorError; - -#[derive(Default, Debug)] -pub struct ParsedUri { - pub user_id: PubkyId, - pub post_id: Option, - pub follow_id: Option, - pub muted_id: Option, - pub bookmark_id: Option, - pub tag_id: Option, - pub file_id: Option, -} - -impl TryFrom<&str> for ParsedUri { - type Error = DynError; - - fn try_from(uri: &str) -> Result { - let mut parsed_uri = ParsedUri::default(); - - // Ensure the URI starts with the correct prefix - if !uri.starts_with("pubky://") { - return Err(EventProcessorError::InvalidEventLine { - message: format!("Invalid URI, must start with pubky://, {}", uri), - } - .into()); - } - - // Extract the user_id from the initial part of the URI - if let Some(user_id) = extract_segment(uri, "pubky://", "/pub/") { - parsed_uri.user_id = PubkyId::try_from(user_id)?; - } else { - return Err(EventProcessorError::InvalidEventLine { - message: format!("Uri Pubky ID is invalid, {}", uri), - } - .into()); - } - - // Ensure that the URI belongs to pubky.app - if let Some(app_segment) = extract_segment(uri, "/pub/", "/") { - if app_segment != "pubky.app" { - return Err(EventProcessorError::InvalidEventLine { - message: format!("The Event URI does not belong to pubky.app, {}", uri), - } - .into()); - } - } else { - return Err(EventProcessorError::InvalidEventLine { - message: format!("The Event URI is malformed, {}", uri), - } - .into()); - } - - // Extract post_id if present - parsed_uri.post_id = extract_segment(uri, "/posts/", "/") - .filter(|id| !id.is_empty()) - .map(String::from); - - // Extract follow_id if present - parsed_uri.follow_id = extract_segment(uri, "/follows/", "/") - .map(PubkyId::try_from) - .transpose() - .map_err(|e| EventProcessorError::InvalidEventLine { - message: format!("{}, {}", e, uri), - })?; - - // Extract muted_id if present - parsed_uri.muted_id = extract_segment(uri, "/mutes/", "/") - .map(PubkyId::try_from) - .transpose() - .map_err(|e| EventProcessorError::InvalidEventLine { - message: format!("{}, {}", e, uri), - })?; - - // Extract bookmark_id if present - parsed_uri.bookmark_id = extract_segment(uri, "/bookmarks/", "/") - .filter(|id| !id.is_empty()) - .map(String::from); - - // Extract tag_id if present - parsed_uri.tag_id = extract_segment(uri, "/tags/", "/") - .filter(|id| !id.is_empty()) - .map(String::from); - - // Extract file_id if present - parsed_uri.file_id = extract_segment(uri, "/files/", "/") - .filter(|id| !id.is_empty()) - .map(String::from); - - Ok(parsed_uri) - } -} - -fn extract_segment<'a>(uri: &'a str, start_pattern: &str, end_pattern: &str) -> Option<&'a str> { - let start_idx = uri.find(start_pattern)? + start_pattern.len(); - let end_idx = uri[start_idx..] - .find(end_pattern) - .map(|i| i + start_idx) - .unwrap_or_else(|| uri.len()); - - Some(&uri[start_idx..end_idx]) -} - -#[cfg(test)] -mod tests { - - use super::*; - - #[test] - fn test_bookmark_uri() { - let uri = - "pubky://phbhg3qgcttn95guepmbud1nzcxhg3xc5j5k4h7i8a4b6wb3nw1o/pub/pubky.app/bookmarks/"; - let parsed_uri = ParsedUri::try_from(uri).unwrap_or_default(); - assert_eq!( - parsed_uri.bookmark_id, None, - "The provided URI has bookmark_id" - ); - } -} From 6ee9c8e80060b97a65d9835789f9f167a514e80b Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 08:40:43 -0400 Subject: [PATCH 29/42] Use PubkyAppObject importer --- Cargo.lock | 14 +++++------ src/events/handlers/bookmark.rs | 11 --------- src/events/handlers/file.rs | 15 +++++------- src/events/handlers/follow.rs | 9 ------- src/events/handlers/mute.rs | 9 ------- src/events/handlers/post.rs | 14 +---------- src/events/handlers/tag.rs | 11 +++++---- src/events/handlers/user.rs | 13 ++-------- src/events/mod.rs | 43 ++++++++++++++++++++++++--------- 9 files changed, 53 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de0ddf3c..4ea40597 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -471,9 +471,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" dependencies = [ "serde", ] @@ -2501,18 +2501,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" dependencies = [ "proc-macro2", "quote", @@ -2730,7 +2730,7 @@ dependencies = [ [[package]] name = "pubky-app-specs" version = "0.3.0" -source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#3ce87b2f8fe92a3985faebbdde49686e64296dcf" +source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#166966ddc395086f2f73aff1dfbc833924c40218" dependencies = [ "base32", "blake3", diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 42deae16..201c939f 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -6,19 +6,8 @@ use crate::models::user::UserCounts; use crate::types::DynError; use chrono::Utc; use log::debug; -use pubky_app_specs::traits::Validatable; use pubky_app_specs::{ParsedUri, PubkyAppBookmark, PubkyId, Resource}; -//TODO: only /posts/ are bookmarkable as of now. -pub async fn put(user_id: PubkyId, bookmark_id: String, blob: &[u8]) -> Result<(), DynError> { - debug!("Indexing new bookmark: {} -> {}", user_id, bookmark_id); - - // Deserialize and validate bookmark - let bookmark = ::try_from(blob, &bookmark_id)?; - - sync_put(user_id, bookmark, bookmark_id).await -} - pub async fn sync_put( user_id: PubkyId, bookmark: PubkyAppBookmark, diff --git a/src/events/handlers/file.rs b/src/events/handlers/file.rs index 43b61ea3..95fe4ad7 100644 --- a/src/events/handlers/file.rs +++ b/src/events/handlers/file.rs @@ -11,30 +11,27 @@ use crate::{ Config, }; use log::{debug, error}; -use pubky_app_specs::{traits::Validatable, PubkyAppFile, PubkyId}; +use pubky_app_specs::{PubkyAppFile, PubkyId}; use tokio::{ fs::{self, remove_file, File}, io::AsyncWriteExt, }; -pub async fn put( +pub async fn sync_put( + file: PubkyAppFile, uri: String, user_id: PubkyId, file_id: String, - blob: &[u8], ) -> Result<(), DynError> { debug!("Indexing new file resource at {}/{}", user_id, file_id); - // Serialize and validate - let file_input = ::try_from(blob, &file_id)?; + debug!("file input {:?}", file); - debug!("file input {:?}", file_input); - - let file_meta = ingest(&user_id, file_id.as_str(), &file_input).await?; + let file_meta = ingest(&user_id, file_id.as_str(), &file).await?; // Create FileDetails object let file_details = - FileDetails::from_homeserver(&file_input, uri, user_id.to_string(), file_id, file_meta); + FileDetails::from_homeserver(&file, uri, user_id.to_string(), file_id, file_meta); // save new file into the Graph file_details.put_to_graph().await?; diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index d629f2e3..ae3ae2e3 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -8,15 +8,6 @@ use crate::types::DynError; use log::debug; use pubky_app_specs::PubkyId; -pub async fn put(follower_id: PubkyId, followee_id: PubkyId, _blob: &[u8]) -> Result<(), DynError> { - debug!("Indexing new follow: {} -> {}", follower_id, followee_id); - - // TODO: in case we want to validate the content of this homeserver object or its `created_at` timestamp - // let _follow = ::try_from(&blob, &followee_id).await?; - - sync_put(follower_id, followee_id).await -} - pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), DynError> { // SAVE TO GRAPH // (follower_id)-[:FOLLOWS]->(followee_id) diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index 3bdf1af5..d90699b6 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -5,15 +5,6 @@ use crate::types::DynError; use log::debug; use pubky_app_specs::PubkyId; -pub async fn put(user_id: PubkyId, muted_id: PubkyId, _blob: &[u8]) -> Result<(), DynError> { - debug!("Indexing new mute: {} -> {}", user_id, muted_id); - - // TODO: in case we want to validate the content of this homeserver object or its `created_at` timestamp - // let _mute = ::try_from(&blob, &muted_id).await?; - - sync_put(user_id, muted_id).await -} - pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { // (user_id)-[:MUTED]->(muted_id) match Muted::put_to_graph(&user_id, &muted_id).await? { diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index d33c7488..c7b2a4cf 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -11,22 +11,10 @@ use crate::queries::get::post_is_safe_to_delete; use crate::types::DynError; use crate::{queries, RedisOps, ScoreAction}; use log::debug; -use pubky_app_specs::{ - traits::Validatable, ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource, -}; +use pubky_app_specs::{ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource}; use super::utils::post_relationships_is_reply; -pub async fn put(author_id: PubkyId, post_id: String, blob: &[u8]) -> Result<(), DynError> { - // Process Post resource and update the databases - debug!("Indexing new post: {}/{}", author_id, post_id); - - // Serialize and validate - let post = ::try_from(blob, &post_id)?; - - sync_put(post, author_id, post_id).await -} - pub async fn sync_put( post: PubkyAppPost, author_id: PubkyId, diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 5d13868d..9f3efa83 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -14,16 +14,17 @@ use crate::ScoreAction; use chrono::Utc; use log::debug; use pubky_app_specs::Resource; -use pubky_app_specs::{traits::Validatable, ParsedUri, PubkyAppTag, PubkyId}; +use pubky_app_specs::{ParsedUri, PubkyAppTag, PubkyId}; use super::utils::post_relationships_is_reply; -pub async fn put(tagger_id: PubkyId, tag_id: String, blob: &[u8]) -> Result<(), DynError> { +pub async fn sync_put( + tag: PubkyAppTag, + tagger_id: PubkyId, + tag_id: String, +) -> Result<(), DynError> { debug!("Indexing new tag: {} -> {}", tagger_id, tag_id); - // Deserialize and validate tag - let tag = ::try_from(blob, &tag_id)?; - // Parse the embeded URI to extract author_id and post_id using parse_tagged_post_uri let parsed_uri = ParsedUri::try_from(tag.uri.as_str())?; let indexed_at = Utc::now().timestamp_millis(); diff --git a/src/events/handlers/user.rs b/src/events/handlers/user.rs index f01b5e9c..726a6f84 100644 --- a/src/events/handlers/user.rs +++ b/src/events/handlers/user.rs @@ -8,19 +8,10 @@ use crate::models::{ use crate::queries::get::user_is_safe_to_delete; use crate::types::DynError; use log::debug; -use pubky_app_specs::{traits::Validatable, PubkyAppUser, PubkyId}; - -pub async fn put(user_id: PubkyId, blob: &[u8]) -> Result<(), DynError> { - // Process profile.json and update the databases - debug!("Indexing new user profile: {}", user_id); - - // Serialize and validate - let user = ::try_from(blob, &user_id)?; - - sync_put(user, user_id).await -} +use pubky_app_specs::{PubkyAppUser, PubkyId}; pub async fn sync_put(user: PubkyAppUser, user_id: PubkyId) -> Result<(), DynError> { + debug!("Indexing new user profile: {}", user_id); // Create UserDetails object let user_details = UserDetails::from_homeserver(user, &user_id).await?; user_details diff --git a/src/events/mod.rs b/src/events/mod.rs index e41dafb4..282ce0ff 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -1,7 +1,7 @@ use crate::{db::connectors::pubky::PubkyConnector, types::DynError}; use error::EventProcessorError; use log::debug; -use pubky_app_specs::{ParsedUri, Resource}; +use pubky_app_specs::{ParsedUri, PubkyAppObject, Resource}; use serde::{Deserialize, Serialize}; use std::fmt; @@ -100,21 +100,40 @@ impl Event { } // drop the pubky_client lock let blob = response.bytes().await?; + let resource = self.parsed_uri.resource; + + // Use the new importer from pubky-app-specs + let pubky_object = PubkyAppObject::from_resource(&resource, &blob).map_err(|e| { + EventProcessorError::PubkyClientError { + message: format!( + "The importer could not create PubkyAppObject from Uri and Blob: {}", + e + ), + } + })?; let user_id = self.parsed_uri.user_id; - match self.parsed_uri.resource { - Resource::User => handlers::user::put(user_id, &blob).await?, - Resource::Post(post_id) => handlers::post::put(user_id, post_id, &blob).await?, - Resource::Follow(followee_id) => { - handlers::follow::put(user_id, followee_id, &blob).await? + match (pubky_object, resource) { + (PubkyAppObject::User(user), Resource::User) => { + handlers::user::sync_put(user, user_id).await? } - Resource::Mute(muted_id) => handlers::mute::put(user_id, muted_id, &blob).await?, - Resource::Bookmark(bookmark_id) => { - handlers::bookmark::put(user_id, bookmark_id, &blob).await? + (PubkyAppObject::Post(post), Resource::Post(post_id)) => { + handlers::post::sync_put(post, user_id, post_id).await? + } + (PubkyAppObject::Follow(_follow), Resource::Follow(followee_id)) => { + handlers::follow::sync_put(user_id, followee_id).await? + } + (PubkyAppObject::Mute(_mute), Resource::Mute(muted_id)) => { + handlers::mute::sync_put(user_id, muted_id).await? + } + (PubkyAppObject::Bookmark(bookmark), Resource::Bookmark(bookmark_id)) => { + handlers::bookmark::sync_put(user_id, bookmark, bookmark_id).await? + } + (PubkyAppObject::Tag(tag), Resource::Tag(tag_id)) => { + handlers::tag::sync_put(tag, user_id, tag_id).await? } - Resource::Tag(tag_id) => handlers::tag::put(user_id, tag_id, &blob).await?, - Resource::File(file_id) => { - handlers::file::put(self.uri, user_id, file_id, &blob).await? + (PubkyAppObject::File(file), Resource::File(file_id)) => { + handlers::file::sync_put(file, self.uri, user_id, file_id).await? } other => { log::debug!("Event type not handled, Resource: {:?}", other); From 1120617b831b700c00ab03b358b4da7b02a5e961 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 09:17:26 -0400 Subject: [PATCH 30/42] Fix re-introduce debug logs --- src/events/handlers/bookmark.rs | 1 + src/events/handlers/follow.rs | 1 + src/events/handlers/mute.rs | 1 + src/events/handlers/post.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/src/events/handlers/bookmark.rs b/src/events/handlers/bookmark.rs index 201c939f..896f1e15 100644 --- a/src/events/handlers/bookmark.rs +++ b/src/events/handlers/bookmark.rs @@ -13,6 +13,7 @@ pub async fn sync_put( bookmark: PubkyAppBookmark, id: String, ) -> Result<(), DynError> { + debug!("Indexing new bookmark: {} -> {}", user_id, id); // Parse the URI to extract author_id and post_id using the updated parse_post_uri let parsed_uri = ParsedUri::try_from(bookmark.uri.as_str())?; let author_id = parsed_uri.user_id; diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index ae3ae2e3..6cad2617 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -9,6 +9,7 @@ use log::debug; use pubky_app_specs::PubkyId; pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), DynError> { + debug!("Indexing new follow: {} -> {}", follower_id, followee_id); // SAVE TO GRAPH // (follower_id)-[:FOLLOWS]->(followee_id) match Followers::put_to_graph(&follower_id, &followee_id).await? { diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index d90699b6..7fac09fb 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -6,6 +6,7 @@ use log::debug; use pubky_app_specs::PubkyId; pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { + debug!("Indexing new mute: {} -> {}", user_id, muted_id); // (user_id)-[:MUTED]->(muted_id) match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index c7b2a4cf..f95af380 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -20,6 +20,7 @@ pub async fn sync_put( author_id: PubkyId, post_id: String, ) -> Result<(), DynError> { + debug!("Indexing new post: {}/{}", author_id, post_id); // Create PostDetails object let post_details = PostDetails::from_homeserver(post.clone(), &author_id, &post_id).await?; // We avoid indexing replies into global feed sorted sets From 6c9e9b8fb12ef796bd6a589fa7016618c20c1021 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 09:31:16 -0400 Subject: [PATCH 31/42] Send invalid line error on unknown parsed resource --- src/events/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/events/mod.rs b/src/events/mod.rs index 282ce0ff..ef248707 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -66,6 +66,13 @@ impl Event { } })?; + if parsed_uri.resource == Resource::Unknown { + return Err(EventProcessorError::InvalidEventLine { + message: format!("Unknown resource in URI: {}", uri), + } + .into()); + } + Ok(Some(Event { uri, event_type, From d0a3607fb272ac7060ffa57c1a2f54a2e13acb36 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 09:46:52 -0400 Subject: [PATCH 32/42] Use uri parser for generate_index_key --- src/events/retry/event.rs | 40 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index 162ced5a..e6f0571a 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use chrono::Utc; +use pubky_app_specs::{ParsedUri, Resource}; use serde::{Deserialize, Serialize}; use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; @@ -7,9 +8,6 @@ use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; pub const RETRY_MAMAGER_PREFIX: &str = "RetryManager"; pub const RETRY_MANAGER_EVENTS_INDEX: [&str; 1] = ["events"]; pub const RETRY_MANAGER_STATE_INDEX: [&str; 1] = ["state"]; -pub const HOMESERVER_PROTOCOL: &str = "pubky:"; -pub const HOMESERVER_PUBLIC_REPOSITORY: &str = "pub"; -pub const HOMESERVER_APP_REPOSITORY: &str = "pubky.app"; /// Represents an event in the retry queue and it is used to manage events that have failed /// to process and need to be retried @@ -42,26 +40,24 @@ impl RetryEvent { /// # Parameters /// - `event_uri`: A string slice representing the event URI to be processed pub fn generate_index_key(event_uri: &str) -> Option { - let parts: Vec<&str> = event_uri.split('/').collect(); - // Ensure the URI structure matches the expected format - if parts.first() != Some(&HOMESERVER_PROTOCOL) - || parts.get(3) != Some(&HOMESERVER_PUBLIC_REPOSITORY) - || parts.get(4) != Some(&HOMESERVER_APP_REPOSITORY) - { - return None; - } + let parsed_uri = match ParsedUri::try_from(event_uri) { + Ok(parsed_uri) => parsed_uri, + Err(_) => return None, + }; - match parts.as_slice() { - // Regular PubkyApp URIs - [_, _, pubky_id, _, _, domain, event_id] => { - Some(format!("{}:{}:{}", pubky_id, domain, event_id)) - } - // PubkyApp user profile URI (profile.json) - [_, _, pubky_id, _, _, "profile.json"] => { - Some(format!("{}:user:profile.json", pubky_id)) - } - _ => None, - } + let user_id = parsed_uri.user_id; + let key = match parsed_uri.resource { + Resource::User => [&user_id, "user"].join(":"), + Resource::Post(id) => [&user_id, "post", &id].join(":"), + Resource::Bookmark(id) => [&user_id, "bookmark", &id].join(":"), + Resource::Tag(id) => [&user_id, "tag", &id].join(":"), + Resource::File(id) => [&user_id, "file", &id].join(":"), + Resource::Follow(id) => [&user_id, "follow", &id].join(":"), + Resource::Mute(id) => [&user_id, "mute", &id].join(":"), + _ => return None, + }; + + Some(key) } /// Stores an event in both a sorted set and a JSON index in Redis. From 8405c53450b548b51a151670df05cc4cbb7f0017 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 10:56:32 -0400 Subject: [PATCH 33/42] Use resource display to generate index key --- Cargo.lock | 2 +- src/events/retry/event.rs | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ea40597..f624bc4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2730,7 +2730,7 @@ dependencies = [ [[package]] name = "pubky-app-specs" version = "0.3.0" -source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#166966ddc395086f2f73aff1dfbc833924c40218" +source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#d64d1e342413404ebe2077dbb821037bdc811db0" dependencies = [ "base32", "blake3", diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index e6f0571a..2c7bcc40 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use chrono::Utc; -use pubky_app_specs::{ParsedUri, Resource}; +use pubky_app_specs::ParsedUri; use serde::{Deserialize, Serialize}; use crate::{events::error::EventProcessorError, types::DynError, RedisOps}; @@ -46,15 +46,9 @@ impl RetryEvent { }; let user_id = parsed_uri.user_id; - let key = match parsed_uri.resource { - Resource::User => [&user_id, "user"].join(":"), - Resource::Post(id) => [&user_id, "post", &id].join(":"), - Resource::Bookmark(id) => [&user_id, "bookmark", &id].join(":"), - Resource::Tag(id) => [&user_id, "tag", &id].join(":"), - Resource::File(id) => [&user_id, "file", &id].join(":"), - Resource::Follow(id) => [&user_id, "follow", &id].join(":"), - Resource::Mute(id) => [&user_id, "mute", &id].join(":"), - _ => return None, + let key = match parsed_uri.resource.id() { + Some(id) => format!("{}:{}:{}", user_id, parsed_uri.resource, id), + None => format!("{}:{}", user_id, parsed_uri.resource), }; Some(key) From 6cc28e0b23c45a8470331e7b140ab17efca3a5a6 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Mon, 3 Feb 2025 12:33:18 -0400 Subject: [PATCH 34/42] Remove hardcoded index keys --- src/events/handlers/follow.rs | 11 ++++++++--- src/events/handlers/mute.rs | 12 +++++++++--- src/events/handlers/post.rs | 10 ++++++++-- src/events/handlers/tag.rs | 13 +++++++++---- tests/watcher/follows/retry_follow.rs | 6 +++--- tests/watcher/mutes/retry_mute.rs | 6 +++--- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/events/handlers/follow.rs b/src/events/handlers/follow.rs index 6cad2617..95445e9b 100644 --- a/src/events/handlers/follow.rs +++ b/src/events/handlers/follow.rs @@ -1,12 +1,13 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; +use crate::events::retry::event::RetryEvent; use crate::models::follow::{Followers, Following, Friends, UserFollows}; use crate::models::notification::Notification; use crate::models::user::UserCounts; use crate::types::DynError; use log::debug; -use pubky_app_specs::PubkyId; +use pubky_app_specs::{user_uri_builder, PubkyId}; pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), DynError> { debug!("Indexing new follow: {} -> {}", follower_id, followee_id); @@ -16,8 +17,12 @@ pub async fn sync_put(follower_id: PubkyId, followee_id: PubkyId) -> Result<(), // Do not duplicate the follow relationship OperationOutcome::Updated => return Ok(()), OperationOutcome::MissingDependency => { - let dependency = vec![format!("{followee_id}:user:profile.json")]; - return Err(EventProcessorError::MissingDependency { dependency }.into()); + if let Some(key) = + RetryEvent::generate_index_key(&user_uri_builder(followee_id.to_string())) + { + let dependency = vec![key]; + return Err(EventProcessorError::MissingDependency { dependency }.into()); + } } // The relationship did not exist, create all related indexes OperationOutcome::CreatedOrDeleted => { diff --git a/src/events/handlers/mute.rs b/src/events/handlers/mute.rs index 7fac09fb..bfff2efe 100644 --- a/src/events/handlers/mute.rs +++ b/src/events/handlers/mute.rs @@ -1,9 +1,10 @@ use crate::db::graph::exec::OperationOutcome; use crate::events::error::EventProcessorError; +use crate::events::retry::event::RetryEvent; use crate::models::user::Muted; use crate::types::DynError; use log::debug; -use pubky_app_specs::PubkyId; +use pubky_app_specs::{user_uri_builder, PubkyId}; pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynError> { debug!("Indexing new mute: {} -> {}", user_id, muted_id); @@ -11,8 +12,13 @@ pub async fn sync_put(user_id: PubkyId, muted_id: PubkyId) -> Result<(), DynErro match Muted::put_to_graph(&user_id, &muted_id).await? { OperationOutcome::Updated => Ok(()), OperationOutcome::MissingDependency => { - let dependency = vec![format!("{muted_id}:user:profile.json")]; - Err(EventProcessorError::MissingDependency { dependency }.into()) + match RetryEvent::generate_index_key(&user_uri_builder(muted_id.to_string())) { + Some(key) => { + let dependency = vec![key]; + Err(EventProcessorError::MissingDependency { dependency }.into()) + } + None => Err("Could not generate missing dependency key".into()), + } } OperationOutcome::CreatedOrDeleted => { Muted(vec![muted_id.to_string()]) diff --git a/src/events/handlers/post.rs b/src/events/handlers/post.rs index f95af380..b2ece67a 100644 --- a/src/events/handlers/post.rs +++ b/src/events/handlers/post.rs @@ -11,7 +11,9 @@ use crate::queries::get::post_is_safe_to_delete; use crate::types::DynError; use crate::{queries, RedisOps, ScoreAction}; use log::debug; -use pubky_app_specs::{ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource}; +use pubky_app_specs::{ + user_uri_builder, ParsedUri, PubkyAppPost, PubkyAppPostKind, PubkyId, Resource, +}; use super::utils::post_relationships_is_reply; @@ -46,7 +48,11 @@ pub async fn sync_put( dependency.push(reply_dependency); } if dependency.is_empty() { - dependency.push(format!("{author_id}:user:profile.json")) + if let Some(key) = + RetryEvent::generate_index_key(&user_uri_builder(author_id.to_string())) + { + dependency.push(key); + } } return Err(EventProcessorError::MissingDependency { dependency }.into()); } diff --git a/src/events/handlers/tag.rs b/src/events/handlers/tag.rs index 9f3efa83..01fcf85c 100644 --- a/src/events/handlers/tag.rs +++ b/src/events/handlers/tag.rs @@ -1,6 +1,7 @@ use crate::db::graph::exec::OperationOutcome; use crate::db::kv::index::json::JsonAction; use crate::events::error::EventProcessorError; +use crate::events::retry::event::RetryEvent; use crate::models::notification::Notification; use crate::models::post::{PostCounts, PostStream}; use crate::models::tag::post::TagPost; @@ -13,7 +14,7 @@ use crate::types::DynError; use crate::ScoreAction; use chrono::Utc; use log::debug; -use pubky_app_specs::Resource; +use pubky_app_specs::{user_uri_builder, Resource}; use pubky_app_specs::{ParsedUri, PubkyAppTag, PubkyId}; use super::utils::post_relationships_is_reply; @@ -164,9 +165,13 @@ async fn put_sync_user( { OperationOutcome::Updated => Ok(()), OperationOutcome::MissingDependency => { - // Ensure that dependencies follow the same format as the RetryManager keys - let dependency = vec![format!("{tagged_user_id}:user:profile.json")]; - Err(EventProcessorError::MissingDependency { dependency }.into()) + match RetryEvent::generate_index_key(&user_uri_builder(tagged_user_id.to_string())) { + Some(key) => { + let dependency = vec![key]; + Err(EventProcessorError::MissingDependency { dependency }.into()) + } + None => Err("Could not generate missing dependency key".into()), + } } OperationOutcome::CreatedOrDeleted => { // SAVE TO INDEX diff --git a/tests/watcher/follows/retry_follow.rs b/tests/watcher/follows/retry_follow.rs index cdbd6e8b..f4810afa 100644 --- a/tests/watcher/follows/retry_follow.rs +++ b/tests/watcher/follows/retry_follow.rs @@ -1,6 +1,6 @@ use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use pubky_app_specs::PubkyAppUser; +use pubky_app_specs::{user_uri_builder, PubkyAppUser}; use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; @@ -46,12 +46,12 @@ async fn test_homeserver_follow_cannot_index() -> Result<()> { assert_eq!(event_state.retry_count, 0); - let dependency_uri = format!("{followee_id}:user:profile.json"); + let dependency_key = RetryEvent::generate_index_key(&user_uri_builder(followee_id.to_string())); match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_uri) + assert_eq!(dependency[0], dependency_key.unwrap()) } _ => panic!("The error type has to be MissingDependency type"), }; diff --git a/tests/watcher/mutes/retry_mute.rs b/tests/watcher/mutes/retry_mute.rs index ef3062ac..8bf83249 100644 --- a/tests/watcher/mutes/retry_mute.rs +++ b/tests/watcher/mutes/retry_mute.rs @@ -1,6 +1,6 @@ use crate::watcher::utils::watcher::{assert_eventually_exists, WatcherTest}; use anyhow::Result; -use pubky_app_specs::PubkyAppUser; +use pubky_app_specs::{user_uri_builder, PubkyAppUser}; use pubky_common::crypto::Keypair; use pubky_nexus::events::{error::EventProcessorError, retry::event::RetryEvent, EventType}; @@ -48,12 +48,12 @@ async fn test_homeserver_mute_cannot_index() -> Result<()> { assert_eq!(event_state.retry_count, 0); - let dependency_uri = format!("{mutee_id}:user:profile.json"); + let dependency_key = RetryEvent::generate_index_key(&user_uri_builder(mutee_id.to_string())); match event_state.error_type { EventProcessorError::MissingDependency { dependency } => { assert_eq!(dependency.len(), 1); - assert_eq!(dependency[0], dependency_uri) + assert_eq!(dependency[0], dependency_key.unwrap()) } _ => panic!("The error type has to be MissingDependency type"), }; From 25efee98766ab7a9252a50ce0601bffea043aa20 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 4 Feb 2025 06:26:32 +0100 Subject: [PATCH 35/42] last changes before merge --- src/events/processor.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/events/processor.rs b/src/events/processor.rs index f0cc7b2f..6bedc0f8 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -13,9 +13,9 @@ use log::{debug, error, info, warn}; use pubky_app_specs::PubkyId; use tokio::{sync::mpsc, time::Duration}; -// REVIEW: This could be env variables and can be part of EventProcessor +// TODO: This could be env variables and can be part of EventProcessor const MAX_ATTEMPTS: i32 = 5; -const MESSAGE_SEND_RETRY: u64 = 10; +const SEND_RETRY_MESSAGE_DELAY: u64 = 10; const RETRY_THRESHOLD: usize = 20; pub struct EventProcessor { @@ -156,7 +156,7 @@ impl EventProcessor { /// - `event`: The event to be processed async fn handle_event(&mut self, event: Event) -> Result<(), DynError> { let mut attempts = 0; - let mut delay = Duration::from_millis(MESSAGE_SEND_RETRY); + let mut delay = Duration::from_millis(SEND_RETRY_MESSAGE_DELAY); if let Err(e) = event.clone().handle().await { if let Some((index_key, retry_event)) = extract_retry_event_info(&event, e) { From b64082a5ae8c56f8725dc2de425bb9777a314bd6 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 4 Feb 2025 06:45:44 +0100 Subject: [PATCH 36/42] fix gh action integration test command --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2f59875..863a4420 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,7 +69,7 @@ jobs: sleep 10 # Give the service a moment to start - name: Run integration tests - run: cargo nextest run -no-fail-fast + run: cargo nextest run --no-fail-fast - name: Show service logs if tests fail if: failure() From 4f3ca172e7220e8aec3abd8b1504cfd357d3c004 Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Tue, 4 Feb 2025 07:48:41 -0400 Subject: [PATCH 37/42] Remove unneeded pubky_client block and drop comment --- src/events/mod.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/events/mod.rs b/src/events/mod.rs index ef248707..5e52fc3d 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -92,19 +92,16 @@ impl Event { pub async fn handle_put_event(self) -> Result<(), DynError> { log::debug!("Handling PUT event for URI: {}", self.uri); - let response; - { - let pubky_client = PubkyConnector::get_pubky_client()?; - response = match pubky_client.get(&self.uri).send().await { - Ok(response) => response, - Err(e) => { - return Err(EventProcessorError::PubkyClientError { - message: format!("{}", e), - } - .into()) + let pubky_client = PubkyConnector::get_pubky_client()?; + let response = match pubky_client.get(&self.uri).send().await { + Ok(response) => response, + Err(e) => { + return Err(EventProcessorError::PubkyClientError { + message: format!("{}", e), } - }; - } // drop the pubky_client lock + .into()) + } + }; let blob = response.bytes().await?; let resource = self.parsed_uri.resource; From ec58a8ec311c112a018a71d8c992e933ab1eedeb Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Tue, 4 Feb 2025 07:49:24 -0400 Subject: [PATCH 38/42] Pin default specs branch --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f624bc4e..29f453e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,9 +580,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.27" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" dependencies = [ "clap_builder", "clap_derive", @@ -602,9 +602,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", @@ -2730,7 +2730,7 @@ dependencies = [ [[package]] name = "pubky-app-specs" version = "0.3.0" -source = "git+https://github.com/pubky/pubky-app-specs?branch=feat%2Furi-parser#d64d1e342413404ebe2077dbb821037bdc811db0" +source = "git+https://github.com/pubky/pubky-app-specs#7300263e79a610725800a2ec17c1c200b1c87594" dependencies = [ "base32", "blake3", @@ -4536,9 +4536,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 6392cb52..b8700d12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ bytes = "1.9.0" # Enforce this version as Pubky and Axum conflict v1.7.1 vs v1.9 pkarr = { git = "https://github.com/pubky/pkarr", branch = "v3-rc1", package = "pkarr" } mainline = { git = "https://github.com/pubky/mainline", branch = "v5-rc1", default-features = false } pubky = { git = "https://github.com/pubky/pubky", branch = "v0.4.0-rc1" } -pubky-app-specs = { git = "https://github.com/pubky/pubky-app-specs", branch = "feat/uri-parser", features = [ +pubky-app-specs = { git = "https://github.com/pubky/pubky-app-specs", features = [ "openapi", ] } tokio = { version = "1.43.0", features = ["full"] } From 253321a11b0156c278535845802abaf24197b59d Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Tue, 4 Feb 2025 07:51:32 -0400 Subject: [PATCH 39/42] Fix comment --- src/events/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/events/mod.rs b/src/events/mod.rs index 5e52fc3d..aa31518e 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -106,7 +106,7 @@ impl Event { let blob = response.bytes().await?; let resource = self.parsed_uri.resource; - // Use the new importer from pubky-app-specs + // Use the importer from pubky-app-specs let pubky_object = PubkyAppObject::from_resource(&resource, &blob).map_err(|e| { EventProcessorError::PubkyClientError { message: format!( From 8bf6997469c96d5d8d56acd63bea9eb10a341f0a Mon Sep 17 00:00:00 2001 From: SHAcollision Date: Tue, 4 Feb 2025 08:13:15 -0400 Subject: [PATCH 40/42] Re introduce scope for Arc --- src/events/mod.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/events/mod.rs b/src/events/mod.rs index aa31518e..ef248707 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -92,21 +92,24 @@ impl Event { pub async fn handle_put_event(self) -> Result<(), DynError> { log::debug!("Handling PUT event for URI: {}", self.uri); - let pubky_client = PubkyConnector::get_pubky_client()?; - let response = match pubky_client.get(&self.uri).send().await { - Ok(response) => response, - Err(e) => { - return Err(EventProcessorError::PubkyClientError { - message: format!("{}", e), + let response; + { + let pubky_client = PubkyConnector::get_pubky_client()?; + response = match pubky_client.get(&self.uri).send().await { + Ok(response) => response, + Err(e) => { + return Err(EventProcessorError::PubkyClientError { + message: format!("{}", e), + } + .into()) } - .into()) - } - }; + }; + } // drop the pubky_client lock let blob = response.bytes().await?; let resource = self.parsed_uri.resource; - // Use the importer from pubky-app-specs + // Use the new importer from pubky-app-specs let pubky_object = PubkyAppObject::from_resource(&resource, &blob).map_err(|e| { EventProcessorError::PubkyClientError { message: format!( From 99ee403de062bcbf3c72ad2be494b4e84e7d0b79 Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 4 Feb 2025 15:08:23 +0100 Subject: [PATCH 41/42] remove mpsc communication channel. Unnecessary --- benches/watcher.rs | 19 ++--- examples/from_file.rs | 6 +- src/events/processor.rs | 128 ++------------------------------- src/events/retry/event.rs | 2 +- src/events/retry/manager.rs | 115 ----------------------------- src/events/retry/mod.rs | 1 - src/watcher.rs | 6 +- tests/watcher/utils/watcher.rs | 6 +- 8 files changed, 17 insertions(+), 266 deletions(-) delete mode 100644 src/events/retry/manager.rs diff --git a/benches/watcher.rs b/benches/watcher.rs index 39f7f6bb..536c506d 100644 --- a/benches/watcher.rs +++ b/benches/watcher.rs @@ -4,10 +4,7 @@ use pubky::Client; use pubky_app_specs::{PubkyAppUser, PubkyAppUserLink}; use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; -use pubky_nexus::{ - events::retry::manager::{RetryManager, WeakRetryManagerSenderChannel}, - EventProcessor, -}; +use pubky_nexus::EventProcessor; use setup::run_setup; use std::time::Duration; use tokio::runtime::Runtime; @@ -19,7 +16,7 @@ mod setup; /// 2. Sign up the user /// 3. Upload a profile.json /// 4. Delete the profile.json -async fn create_homeserver_with_events() -> (Testnet, String, WeakRetryManagerSenderChannel) { +async fn create_homeserver_with_events() -> (Testnet, String) { // Create the test environment let testnet = Testnet::new(3).unwrap(); let homeserver = Homeserver::start_test(&testnet).await.unwrap(); @@ -30,9 +27,6 @@ async fn create_homeserver_with_events() -> (Testnet, String, WeakRetryManagerSe let keypair = Keypair::random(); let user_id = keypair.public_key().to_z32(); - // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager - let sender_channel = RetryManager::clone_sender_channel(); - // Create and delete a user profile (as per your requirement) client .signup(&keypair, &homeserver.public_key()) @@ -64,7 +58,7 @@ async fn create_homeserver_with_events() -> (Testnet, String, WeakRetryManagerSe // Delete the user profile client.delete(url.as_str()).send().await.unwrap(); - (testnet, homeserver_url, sender_channel) + (testnet, homeserver_url) } fn bench_create_delete_user(c: &mut Criterion) { @@ -76,16 +70,15 @@ fn bench_create_delete_user(c: &mut Criterion) { // Set up the environment only once let rt = Runtime::new().unwrap(); - let (_, homeserver_url, sender) = rt.block_on(create_homeserver_with_events()); + let (_, homeserver_url) = rt.block_on(create_homeserver_with_events()); c.bench_function("create_delete_homeserver_user", |b| { b.to_async(&rt).iter(|| { - let sender_clone = sender.clone(); // Clone the sender for each iteration + // Clone the sender for each iteration let homeserver_url_clone = homeserver_url.clone(); async move { // Benchmark the event processor initialization and run - let mut event_processor = - EventProcessor::test(homeserver_url_clone, sender_clone).await; + let mut event_processor = EventProcessor::test(homeserver_url_clone).await; event_processor.run().await.unwrap(); } }); diff --git a/examples/from_file.rs b/examples/from_file.rs index 24095916..5b7e57bc 100644 --- a/examples/from_file.rs +++ b/examples/from_file.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::{setup, types::DynError, Config, EventProcessor}; use std::fs::File; use std::io::{self, BufRead}; @@ -15,10 +14,7 @@ async fn main() -> Result<(), DynError> { let config = Config::from_env(); setup(&config).await; - // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager - let sender_channel = RetryManager::clone_sender_channel(); - - let mut event_processor = EventProcessor::from_config(&config, sender_channel).await?; + let mut event_processor = EventProcessor::from_config(&config).await?; let events = read_events_from_file().unwrap(); event_processor.process_event_lines(events).await?; diff --git a/src/events/processor.rs b/src/events/processor.rs index 6bedc0f8..a0e21f7c 100644 --- a/src/events/processor.rs +++ b/src/events/processor.rs @@ -1,34 +1,18 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; - use super::error::EventProcessorError; -use super::retry::manager::{RetryQueueMessage, WeakRetryManagerSenderChannel}; use super::Event; use crate::events::retry::event::RetryEvent; -use crate::events::retry::manager::RetryManager; use crate::types::DynError; use crate::PubkyConnector; use crate::{models::homeserver::Homeserver, Config}; -use log::{debug, error, info, warn}; +use log::{debug, error, info}; use pubky_app_specs::PubkyId; -use tokio::{sync::mpsc, time::Duration}; - -// TODO: This could be env variables and can be part of EventProcessor -const MAX_ATTEMPTS: i32 = 5; -const SEND_RETRY_MESSAGE_DELAY: u64 = 10; -const RETRY_THRESHOLD: usize = 20; pub struct EventProcessor { pub homeserver: Homeserver, limit: u32, - pub retry_manager_sender_channel: WeakRetryManagerSenderChannel, - consecutive_message_send_failures: Arc, } impl EventProcessor { - pub fn set_sender_channel(&mut self, channel: WeakRetryManagerSenderChannel) { - self.retry_manager_sender_channel = channel; - } /// Creates a new `EventProcessor` instance for testing purposes. /// /// This function initializes an `EventProcessor` configured with: @@ -41,24 +25,16 @@ impl EventProcessor { /// # Parameters /// - `homeserver_id`: A `String` representing the URL of the homeserver to be used in the test environment. /// - `tx`: A `RetryManagerSenderChannel` used to handle outgoing messages or events. - pub async fn test( - homeserver_id: String, - retry_manager_sender_channel: WeakRetryManagerSenderChannel, - ) -> Self { + pub async fn test(homeserver_id: String) -> Self { let id = PubkyId::try_from(&homeserver_id).expect("Homeserver ID should be valid"); let homeserver = Homeserver::new(id).await.unwrap(); Self { homeserver, limit: 1000, - retry_manager_sender_channel, - consecutive_message_send_failures: Arc::new(AtomicUsize::new(0)), } } - pub async fn from_config( - config: &Config, - retry_manager_sender_channel: WeakRetryManagerSenderChannel, - ) -> Result { + pub async fn from_config(config: &Config) -> Result { let homeserver = Homeserver::from_config(config).await?; let limit = config.events_limit; @@ -67,12 +43,7 @@ impl EventProcessor { homeserver ); - Ok(Self { - homeserver, - limit, - retry_manager_sender_channel, - consecutive_message_send_failures: Arc::new(AtomicUsize::new(0)), - }) + Ok(Self { homeserver, limit }) } pub async fn run(&mut self) -> Result<(), DynError> { @@ -155,101 +126,16 @@ impl EventProcessor { /// # Parameters: /// - `event`: The event to be processed async fn handle_event(&mut self, event: Event) -> Result<(), DynError> { - let mut attempts = 0; - let mut delay = Duration::from_millis(SEND_RETRY_MESSAGE_DELAY); - if let Err(e) = event.clone().handle().await { if let Some((index_key, retry_event)) = extract_retry_event_info(&event, e) { - let queue_message = RetryQueueMessage::ProcessEvent(index_key.clone(), retry_event); - - while attempts < MAX_ATTEMPTS { - let mut restart = false; - if let Some(sender_arc) = self.retry_manager_sender_channel.upgrade() { - let mut sender_guard = sender_arc.lock().await; - - if let Some(sender) = sender_guard.as_mut() { - match sender.try_send(queue_message.clone()) { - Err(mpsc::error::TrySendError::Full(_)) => { - warn!( - "Retry channel is full. Retrying in {:?}... (attempt {}/{})", - delay, attempts + 1, MAX_ATTEMPTS - ); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - warn!("Retry channel receiver is unavailable! Restarting RetryManager..."); - restart = true; - } - _ => { - // Message sent successfully, reset failure counter - self.consecutive_message_send_failures - .store(0, Ordering::Relaxed); - return Ok(()); - } - } - } else { - warn!("Retry sender channel has been dropped. Retrying in {:?}... (attempt {}/{})", - delay, attempts + 1, MAX_ATTEMPTS - ); - } - } else { - warn!("Sender is invalid/dropped. Retrying event in next iteration..."); - } - - if restart { - self.update_sender_channel().await; - break; - } - - tokio::time::sleep(delay).await; - // Apply exponential backoff before the next retry attempt - delay *= 2; - attempts += 1; + error!("{}, {}", retry_event.error_type, index_key); + if let Err(err) = retry_event.put_to_index(index_key).await { + error!("Failed to put event to retry index: {}", err); } - - error!( - "Message passing failed: Unable to send event {:?} after {} attempts. Dropping event...", - index_key, MAX_ATTEMPTS - ); - - // Increase failure counter to track consecutive failures - self.consecutive_message_send_failures - .fetch_add(1, Ordering::Relaxed); - self.evaluate_message_send_failures().await; } } Ok(()) } - - /// Checks if the consecutive message send failure count has reached RETRY_THRESHOLD - /// and triggers a RetryManager restart if necessary - /// This prevents RetryManager from restarting too frequently - async fn evaluate_message_send_failures(&mut self) { - // Before reset the retry manager, reset the consecutive message send fail counter - let reset_needed = self - .consecutive_message_send_failures - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { - if count >= RETRY_THRESHOLD { - Some(0) // Reset counter - } else { - None - } - }) - .is_ok(); - - if reset_needed { - warn!( - "Message send failure threshold reached ({} consecutive failures). Restarting RetryManager...", - RETRY_THRESHOLD - ); - self.update_sender_channel().await; - } - } - - /// Restarts the RetryManager and updates the sender channel reference - async fn update_sender_channel(&mut self) { - RetryManager::restart().await; - self.set_sender_channel(RetryManager::clone_sender_channel()); - } } /// Extracts retry-related information from an event and its associated error diff --git a/src/events/retry/event.rs b/src/events/retry/event.rs index 2c7bcc40..62135408 100644 --- a/src/events/retry/event.rs +++ b/src/events/retry/event.rs @@ -91,7 +91,7 @@ impl RetryEvent { /// # Arguments /// * `event_index` - A `&str` representing the event index to retrieve pub async fn get_from_index(event_index: &str) -> Result, DynError> { - let index = &[RETRY_MANAGER_STATE_INDEX, [event_index]].concat(); + let index: &Vec<&str> = &[RETRY_MANAGER_STATE_INDEX, [event_index]].concat(); Self::try_from_index_json(index).await } } diff --git a/src/events/retry/manager.rs b/src/events/retry/manager.rs deleted file mode 100644 index ca8bbf82..00000000 --- a/src/events/retry/manager.rs +++ /dev/null @@ -1,115 +0,0 @@ -use log::{debug, error, info}; -use once_cell::sync::OnceCell; -use std::sync::{Arc, Weak}; -use tokio::sync::mpsc; -use tokio::sync::{ - mpsc::{Receiver, Sender}, - Mutex, -}; - -use super::event::RetryEvent; - -pub const CHANNEL_BUFFER: usize = 1024; -pub static RETRY_MANAGER: OnceCell = OnceCell::new(); - -/// Represents commands sent to the retry manager for handling events in the retry queue -#[derive(Debug, Clone)] -pub enum RetryQueueMessage { - // Command to retry all pending events that are waiting to be indexed - RetryEvent(String), - /// Command to process a specific event in the retry queue - /// This action can either add an event to the queue or use the event as context to remove it from the queue - /// - `String`: The index or identifier of the event being processed - /// - `RetryEvent`: The event to be added to or processed for removal from the retry queue - ProcessEvent(String, RetryEvent), -} - -pub type WeakRetryManagerSenderChannel = Weak>>>; -//type RetryManagerReceiverChannel = Receiver; - -/// Manages the retry queue for processing failed events. -/// -/// The `RetryManager` is responsible for handling messages that involve retrying or -/// processing failed events in an asynchronous event system. It listens for incoming -/// retry-related messages and processes them accordingly -/// -/// ## Responsibilities: -/// - Receives and processes messages related to event retries -/// - Maintains a queue of events that need to be retried -/// - Ensures failed events are reprocessed in an orderly manner -#[derive(Debug, Clone)] -pub struct RetryManager { - sender_channel: Arc>>>, -} - -/// Initializes a new `RetryManager` with a message-passing channel -impl RetryManager { - pub fn initialise() -> &'static RetryManager { - RETRY_MANAGER.get_or_init(|| { - let (tx, rx) = mpsc::channel(CHANNEL_BUFFER); - - tokio::spawn(Self::process_messages(rx)); - - RetryManager { - sender_channel: Arc::new(Mutex::new(Some(tx))), - } - }) - } - - /// Returns a weak reference to the sender channel - /// - /// It clones the sender channel as a `Weak` reference instead of `Arc`, - /// preventing strong ownership cycles that could lead to memory leaks. By using `Weak`, - /// the sender channel can be automatically cleaned up when all strong references - /// (`Arc`) to it are dropped. This ensures that: - /// - /// - The sender channel is **not kept alive indefinitely** if the `RetryManager` is restarted - /// - It allows us to **check if the sender is still valid** before attempting to send messages - /// - Prevents **unnecessary memory usage** by ensuring unused channels are cleaned up - /// - /// # Returns: - /// - A `Weak` reference to the sender channel, which can be upgraded to `Arc` when needed - pub fn clone_sender_channel() -> Weak>>> { - Arc::downgrade(&Self::initialise().sender_channel) - } - - /// Replaces the existing sender channel with a new one and restarts the communication channel - pub async fn restart() { - let (new_sender_channel, new_receiver_channel) = mpsc::channel(CHANNEL_BUFFER); - - { - let mut sender_lock = Self::initialise().sender_channel.lock().await; - *sender_lock = None; // Remove old sender - } - - tokio::spawn(RetryManager::process_messages(new_receiver_channel)); - - let mut sender_lock = Self::initialise().sender_channel.lock().await; - *sender_lock = Some(new_sender_channel.clone()); - - info!("RetryManager restarted the channel communication"); - } - - /// Executes the main event loop to process messages from the channel - /// This function listens for incoming messages on the receiver channel and handles them - /// based on their type - /// The loop runs continuously until the channel is closed, ensuring that all messages - /// are processed appropriately - /// - /// # Arguments - /// * `rx` - The receiver channel for retry messages - async fn process_messages(mut rx: Receiver) { - // Listen all the messages in the channel - while let Some(message) = rx.recv().await { - match message { - RetryQueueMessage::ProcessEvent(index_key, retry_event) => { - error!("{}, {}", retry_event.error_type, index_key); - if let Err(err) = retry_event.put_to_index(index_key).await { - error!("Failed to put event to index: {}", err); - } - } - _ => debug!("New message received, not handled: {:?}", message), - } - } - } -} diff --git a/src/events/retry/mod.rs b/src/events/retry/mod.rs index 7003034b..53f11265 100644 --- a/src/events/retry/mod.rs +++ b/src/events/retry/mod.rs @@ -1,2 +1 @@ pub mod event; -pub mod manager; diff --git a/src/watcher.rs b/src/watcher.rs index 73b556e1..475b74a5 100644 --- a/src/watcher.rs +++ b/src/watcher.rs @@ -1,6 +1,5 @@ use log::error; use log::info; -use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::PubkyConnector; use pubky_nexus::{setup, Config, EventProcessor}; use tokio::time::{sleep, Duration}; @@ -15,11 +14,8 @@ async fn main() -> Result<(), Box> { // Initializes the PubkyConnector with the configuration PubkyConnector::initialise(&config, None).await?; - // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager - let sender_channel = RetryManager::clone_sender_channel(); - // Create and configure the event processor - let mut event_processor = EventProcessor::from_config(&config, sender_channel).await?; + let mut event_processor = EventProcessor::from_config(&config).await?; loop { info!("Fetching events..."); diff --git a/tests/watcher/utils/watcher.rs b/tests/watcher/utils/watcher.rs index 09b04ee5..b208fb02 100644 --- a/tests/watcher/utils/watcher.rs +++ b/tests/watcher/utils/watcher.rs @@ -10,7 +10,6 @@ use pubky_app_specs::{ use pubky_common::crypto::Keypair; use pubky_homeserver::Homeserver; use pubky_nexus::events::retry::event::RetryEvent; -use pubky_nexus::events::retry::manager::RetryManager; use pubky_nexus::events::Event; use pubky_nexus::types::DynError; use pubky_nexus::{setup, Config, EventProcessor, PubkyConnector}; @@ -48,15 +47,12 @@ impl WatcherTest { let homeserver = Homeserver::start_test(&testnet).await?; let homeserver_id = homeserver.public_key().to_string(); - // Initialise the retry manager and prepare the sender channel to send the messages to the retry manager - let sender_channel = RetryManager::clone_sender_channel(); - match PubkyConnector::initialise(&config, Some(&testnet)).await { Ok(_) => debug!("WatcherTest: PubkyConnector initialised"), Err(e) => debug!("WatcherTest: {}", e), } - let event_processor = EventProcessor::test(homeserver_id, sender_channel).await; + let event_processor = EventProcessor::test(homeserver_id).await; Ok(Self { config, From 475f59129cd683bd9fa995a6db95dd7941951b4a Mon Sep 17 00:00:00 2001 From: tipogi Date: Tue, 4 Feb 2025 15:45:57 +0100 Subject: [PATCH 42/42] fmt fix --- tests/watcher/tags/post_del.rs | 2 +- tests/watcher/tags/post_muti_user.rs | 4 ++-- tests/watcher/tags/post_put.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/watcher/tags/post_del.rs b/tests/watcher/tags/post_del.rs index acf733c8..bf75231f 100644 --- a/tests/watcher/tags/post_del.rs +++ b/tests/watcher/tags/post_del.rs @@ -134,7 +134,7 @@ async fn test_homeserver_del_tag_post() -> Result<()> { .await .unwrap(); assert!(tag_timeline.is_none()); - + // Cleanup user and post test.cleanup_post(&author_user_id, &post_id).await?; test.cleanup_user(&author_user_id).await?; diff --git a/tests/watcher/tags/post_muti_user.rs b/tests/watcher/tags/post_muti_user.rs index 2dc4307f..5a457b11 100644 --- a/tests/watcher/tags/post_muti_user.rs +++ b/tests/watcher/tags/post_muti_user.rs @@ -61,7 +61,7 @@ async fn test_homeserver_multi_user() -> Result<()> { let label_water = "water"; let label_fire = "fire"; - + // Step 2: Create tags let mut tag_urls = Vec::with_capacity(5); let water_taggers = [tagger_a_id, tagger_b_id, tagger_c_id]; @@ -190,7 +190,7 @@ async fn test_homeserver_multi_user() -> Result<()> { "Total engagement should be present" ); assert_eq!(total_engagement.unwrap(), 5); - + // Step 4: DEL tag from homeserver for tag_url in tag_urls { test.del(&tag_url).await?; diff --git a/tests/watcher/tags/post_put.rs b/tests/watcher/tags/post_put.rs index 8c7fd78c..33f1d1a1 100644 --- a/tests/watcher/tags/post_put.rs +++ b/tests/watcher/tags/post_put.rs @@ -139,7 +139,7 @@ async fn test_homeserver_put_tag_post() -> Result<()> { 0, "Post author should have 0 notification. Self tagging." ); - + // Cleanup user and post test.cleanup_post(&tagger_user_id, &post_id).await?; test.cleanup_user(&tagger_user_id).await?;