[analytics] add example for offline analysis for exchange data (#6)

* add offline analytics queries

* scaffold matching

* try broad search expanding the ledger funding window

* add caching

* prevent duplicates in impossible

* docs

docs/cql/account_exclusive_trading.cql

@@ -0,0 +1,13 @@
+MATCH (a)-[r:Swap]-(b)
+WITH a, b, count(r) AS ab_count
+WHERE ab_count >= 5
+MATCH (a)-[r_all:Swap]-()
+WITH a, b, ab_count, count(r_all) AS total_a_count
+RETURN
+    a,
+    b,
+    ab_count,
+    total_a_count,
+    (toFloat(ab_count) / total_a_count) * 100 AS exclusivity_percentage
+ORDER BY ab_count DESC, exclusivity_percentage DESC
+LIMIT 100

docs/cql/cycling_coins_between_pairs.cql

@@ -0,0 +1,2 @@
+MATCH p=SHORTEST 1 (a:SwapAccount)-[r:Swap]->(b:SwapAccount)-[r2:Swap]->(a:SwapAccount)
+RETURN p

docs/cql/find_pump_stats.cql

@@ -0,0 +1,14 @@
+MATCH (a)-[r:Swap]-()
+
+WITH a,
+     count(r) AS total_trades,
+     sum(CASE WHEN r.shill_bid = true THEN 1 ELSE 0 END) AS shill_bid_count,
+    sum(CASE WHEN r.price_vs_rms_hour > 1.0 THEN 1 ELSE 0 END) AS pump_count
+WHERE total_trades > 100
+RETURN
+    a,
+    total_trades,
+    shill_bid_count,
+    (toFloat(shill_bid_count) / total_trades) AS shill_bid_percentage,
+    (toFloat(pump_count) / total_trades) AS pump_percentage
+ORDER BY shill_bid_percentage DESC

docs/cql/find_shill_in_range.cql

@@ -0,0 +1,4 @@
+MATCH p=()-[r:Swap {`shill_bid`: TRUE }]->()
+WHERE date(r.filled_at) > date("2024-02-10")
+AND date(r.filled_at) < date("2024-03-02")
+RETURN p

docs/cql/find_top_exchange_depositors.cql

@@ -0,0 +1,19 @@
+WITH "0xf57d3968d0bfd5b3120fda88f34310c70bd72033f77422f4407fbbef7c24557a" AS olswap_deposit
+
+// Step 1: Get the list of all depositors
+MATCH (depositor:Account)-[tx:Tx]->(onboard:Account {address: olswap_deposit})
+WITH COLLECT(DISTINCT depositor) AS all_depositors, olswap_deposit, tx
+
+// Step 2: Match depositors and amounts within the date range
+
+UNWIND all_depositors AS depositor
+
+OPTIONAL MATCH (depositor)-[tx2:Tx]->(onboard:Account {address: olswap_deposit})
+WHERE tx2.block_datetime >= datetime('2024-01-07') AND tx2.block_datetime <= datetime('2024-01-09')
+
+
+RETURN
+  depositor.address AS depositor_address,
+  COALESCE(SUM(tx2.V7_OlAccountTransfer_amount), 0) AS deposit_amount,
+  count(tx2)
+ORDER BY deposit_amount DESC

docs/cql/frequent_exchange_traders.cql

@@ -0,0 +1,6 @@
+MATCH (from:SwapAccount)-[r:Swap]-(to:SwapAccount)
+WITH from, to, COUNT(r) AS transaction_count
+ORDER BY transaction_count DESC
+LIMIT 500
+MATCH p=(from)-[r:Swap]-(to)
+RETURN p, transaction_count

docs/cql/shill_trader_pairs.cql

@@ -0,0 +1,15 @@
+MATCH (a)-[r:Swap]-(b)
+WITH a, b,
+     count(r) AS total_count,
+     sum(CASE WHEN r.shill_bid = true THEN 1 ELSE 0 END) AS shill_bid_count,
+     sum(CASE WHEN r.price_vs_rms_hour > 1 THEN 1 ELSE 0 END) AS price_vs_rms24h_count
+WHERE total_count >= 5
+ORDER BY total_count DESC
+RETURN
+    a,
+    b,
+    total_count,
+    shill_bid_count,
+    (toFloat(shill_bid_count) / total_count) * 100 AS shill_bid_percentage,
+    price_vs_rms24h_count,
+    (toFloat(price_vs_rms24h_count) / total_count) * 100 AS price_vs_rms24h_percentage

docs/cql/top_exchange_funding_required.cql

@@ -0,0 +1,6 @@
+MATCH p=(e:SwapAccount)-[d:DailyLedger]-(ul:UserLedger)
+WHERE d.date < datetime("2024-01-16")
+WITH e.swap_id AS id, max(ul.`total_funded`) as funded
+
+RETURN id, funded
+ORDER BY funded DESCENDING

docs/cql/trace_owner.cql

@@ -0,0 +1,19 @@
+WITH [
+// olswap onboarding
+'0xf57d3968d0bfd5b3120fda88f34310c70bd72033f77422f4407fbbef7c24557a',
+// ignore superspreader
+'0x85b68bdeb3bd8ca47f1cf90dfb332404290afda582c586cb645b3b045b54825b'
+] AS exclude
+
+MATCH p = SHORTEST 1 (o:Owner {alias: 'name'})-[r *..3]->(:SwapAccount)
+WHERE NONE(
+  r IN relationships(p)
+    WHERE r.relation IS NOT NULL
+    AND NOT r.relation IN ["Vouch"]
+  )
+  AND NONE(
+    n IN nodes(p)
+    WHERE n.address IS NOT NULL
+    AND n.address IN exclude
+  )
+RETURN p

docs/local_testing.md

@@ -28,3 +28,17 @@ RETURN COUNT(DISTINCT(r))
 ```
 
 Should return `25450`
+
+# Testing offline analytics
+NOTE: you must have a fully populated DB to run these queries
+
+Replay the funding requirement on an exchange and match to deposits. This is slow.
+```
+cargo r analytics trades-matching --start-day 2024-01-07 --end-day 2024-01-15 --replay-balances 10
+
+```
+
+Match simple dumps
+```
+cargo r analytics trades-matching --start-day 2024-01-07 --end-day 2024-01-15 --match-simple-dumps 1.01
+```

src/analytics/enrich_account_funding.rs

@@ -2,8 +2,6 @@ use anyhow::{Context, Result};
 use chrono::{DateTime, Utc};
 use log::{error, trace};
 use neo4rs::{Graph, Query};
-// use log::trace;
-// use neo4rs::{Graph, Query};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::BTreeMap,
@@ -13,6 +11,9 @@ use std::{
 
 use crate::schema_exchange_orders::ExchangeOrder;
 
+#[cfg(test)]
+use crate::date_util::parse_date;
+
 #[derive(Default, Debug, Clone, Deserialize, Serialize)]
 pub struct AccountDataAlt {
     pub current_balance: f64,
@@ -248,14 +249,6 @@ pub fn generate_cypher_query(map: String) -> String {
     )
 }
 
-/// Helper function to parse "YYYY-MM-DD" into `DateTime<Utc>`
-pub fn parse_date(date_str: &str) -> DateTime<Utc> {
-    let datetime_str = format!("{date_str}T00:00:00Z"); // Append time and UTC offset
-    DateTime::parse_from_rfc3339(&datetime_str)
-        .expect("Invalid date format; expected YYYY-MM-DD")
-        .with_timezone(&Utc)
-}
-
 #[test]
 fn test_replay_transactions() {
     let mut orders = vec![
@@ -364,6 +357,7 @@ fn test_replay_transactions() {
 #[test]
 fn test_example_user() -> Result<()> {
     use crate::extract_exchange_orders;
+
     use std::path::PathBuf;
     let path = env!("CARGO_MANIFEST_DIR");
     let buf = PathBuf::from(path).join("tests/fixtures/savedOlOrders2.json");

src/analytics/mod.rs

@@ -1,3 +1,4 @@
 pub mod enrich_account_funding;
 pub mod enrich_rms;
 pub mod exchange_stats;
+pub mod offline_matching;