From ea5379e0c4f3d696fb09117d27a722998392c9d8 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 19 Nov 2023 21:31:02 -0500 Subject: [PATCH 01/72] fmt --- digital_asset_types/src/dao/scopes/asset.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/digital_asset_types/src/dao/scopes/asset.rs b/digital_asset_types/src/dao/scopes/asset.rs index 58e4cbbaf..2df576591 100644 --- a/digital_asset_types/src/dao/scopes/asset.rs +++ b/digital_asset_types/src/dao/scopes/asset.rs @@ -1,10 +1,7 @@ use crate::{ dao::{ - asset::{self}, - asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, - extensions::{self, instruction::PascalCase}, - sea_orm_active_enums::Instruction, - Cursor, FullAsset, GroupingSize, Pagination, + asset, asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, + extensions, Cursor, FullAsset, GroupingSize, Pagination, }, rpc::filter::AssetSortDirection, }; @@ -465,7 +462,8 @@ pub async fn fetch_transactions( .into_iter() .map(|transaction| { let tx = bs58::encode(transaction.tx).into_string(); - let ix = Instruction::to_pascal_case(&transaction.instruction).to_string(); + let ix = extensions::instruction::PascalCase::to_pascal_case(&transaction.instruction) + .to_string(); (tx, ix) }) .collect(); From 8b12316efa6abf242c3853102fbb256c16f524de Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 19 Nov 2023 22:04:45 -0500 Subject: [PATCH 02/72] clippy --- nft_ingester/src/backfiller.rs | 11 +- .../src/program_transformers/bubblegum/db.rs | 442 ++++++++++++++++++ .../program_transformers/bubblegum/mint_v1.rs | 364 +++++++++++++++ nft_ingester/src/program_transformers/mod.rs | 189 ++++++++ .../src/program_transformers/token/mod.rs | 141 ++++++ .../token_metadata/v1_asset.rs | 393 ++++++++++++++++ 6 files changed, 1532 insertions(+), 8 deletions(-) create mode 100644 nft_ingester/src/program_transformers/bubblegum/db.rs create mode 100644 nft_ingester/src/program_transformers/bubblegum/mint_v1.rs create mode 100644 nft_ingester/src/program_transformers/mod.rs create mode 100644 nft_ingester/src/program_transformers/token/mod.rs create mode 100644 nft_ingester/src/program_transformers/token_metadata/v1_asset.rs diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs index fd7d3e19c..fcf27fad3 100644 --- a/nft_ingester/src/backfiller.rs +++ b/nft_ingester/src/backfiller.rs @@ -628,14 +628,9 @@ impl<'a, T: Messenger> Backfiller<'a, T> { &mut self, btree: &BackfillTree, ) -> Result, IngesterError> { - let address = match Pubkey::try_from(btree.unique_tree.tree.as_slice()) { - Ok(pubkey) => pubkey, - Err(error) => { - return Err(IngesterError::DeserializationError(format!( - "failed to parse pubkey: {error:?}" - ))) - } - }; + let address = Pubkey::try_from(btree.unique_tree.tree.as_slice()).map_err(|error| { + IngesterError::DeserializationError(format!("failed to parse pubkey: {error:?}")) + })?; let slots = self.find_slots_via_address(&address).await?; let address = btree.unique_tree.tree.clone(); diff --git a/nft_ingester/src/program_transformers/bubblegum/db.rs b/nft_ingester/src/program_transformers/bubblegum/db.rs new file mode 100644 index 000000000..c562be2be --- /dev/null +++ b/nft_ingester/src/program_transformers/bubblegum/db.rs @@ -0,0 +1,442 @@ +use crate::error::IngesterError; +use digital_asset_types::dao::{ + asset, asset_creators, asset_grouping, backfill_items, cl_audits, cl_items, +}; +use log::{debug, info}; +use mpl_bubblegum::types::Collection; +use sea_orm::{ + query::*, sea_query::OnConflict, ActiveValue::Set, ColumnTrait, DbBackend, EntityTrait, +}; +use spl_account_compression::events::ChangeLogEventV1; + +pub async fn save_changelog_event<'c, T>( + change_log_event: &ChangeLogEventV1, + slot: u64, + txn_id: &str, + txn: &T, + cl_audits: bool, +) -> Result +where + T: ConnectionTrait + TransactionTrait, +{ + insert_change_log(change_log_event, slot, txn_id, txn, cl_audits).await?; + Ok(change_log_event.seq) +} + +fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { + index - 2i64.pow(tree_height) +} + +pub async fn insert_change_log<'c, T>( + change_log_event: &ChangeLogEventV1, + slot: u64, + txn_id: &str, + txn: &T, + cl_audits: bool, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let mut i: i64 = 0; + let depth = change_log_event.path.len() - 1; + let tree_id = change_log_event.id.as_ref(); + for p in change_log_event.path.iter() { + let node_idx = p.index as i64; + debug!( + "seq {}, index {} level {}, node {:?}, txn: {:?}", + change_log_event.seq, + p.index, + i, + bs58::encode(p.node).into_string(), + txn_id, + ); + let leaf_idx = if i == 0 { + Some(node_idx_to_leaf_idx(node_idx, depth as u32)) + } else { + None + }; + + let item = cl_items::ActiveModel { + tree: Set(tree_id.to_vec()), + level: Set(i), + node_idx: Set(node_idx), + hash: Set(p.node.as_ref().to_vec()), + seq: Set(change_log_event.seq as i64), + leaf_idx: Set(leaf_idx), + ..Default::default() + }; + + let audit_item: Option = if cl_audits { + let mut ai: cl_audits::ActiveModel = item.clone().into(); + ai.tx = Set(txn_id.to_string()); + Some(ai) + } else { + None + }; + + i += 1; + let mut query = cl_items::Entity::insert(item) + .on_conflict( + OnConflict::columns([cl_items::Column::Tree, cl_items::Column::NodeIdx]) + .update_columns([ + cl_items::Column::Hash, + cl_items::Column::Seq, + cl_items::Column::LeafIdx, + cl_items::Column::Level, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!("{} WHERE excluded.seq > cl_items.seq", query.sql); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + // Insert the audit item after the insert into cl_items have been completed + if let Some(audit_item) = audit_item { + cl_audits::Entity::insert(audit_item).exec(txn).await?; + } + } + + // If and only if the entire path of nodes was inserted into the `cl_items` table, then insert + // a single row into the `backfill_items` table. This way if an incomplete path was inserted + // into `cl_items` due to an error, a gap will be created for the tree and the backfiller will + // fix it. + if i - 1 == depth as i64 { + // See if the tree already exists in the `backfill_items` table. + let rows = backfill_items::Entity::find() + .filter(backfill_items::Column::Tree.eq(tree_id)) + .limit(1) + .all(txn) + .await?; + + // If the tree does not exist in `backfill_items` and the sequence number is greater than 1, + // then we know we will need to backfill the tree from sequence number 1 up to the current + // sequence number. So in this case we set at flag to force checking the tree. + let force_chk = rows.is_empty() && change_log_event.seq > 1; + + info!("Adding to backfill_items table at level {}", i - 1); + let item = backfill_items::ActiveModel { + tree: Set(tree_id.to_vec()), + seq: Set(change_log_event.seq as i64), + slot: Set(slot as i64), + force_chk: Set(force_chk), + backfilled: Set(false), + failed: Set(false), + ..Default::default() + }; + + backfill_items::Entity::insert(item).exec(txn).await?; + } + + Ok(()) + //TODO -> set maximum size of path and break into multiple statements +} + +#[allow(clippy::too_many_arguments)] +pub async fn upsert_asset_with_leaf_info( + txn: &T, + id: Vec, + nonce: i64, + tree_id: Vec, + leaf: Vec, + data_hash: [u8; 32], + creator_hash: [u8; 32], + seq: i64, + was_decompressed: bool, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let data_hash = bs58::encode(data_hash).into_string().trim().to_string(); + let creator_hash = bs58::encode(creator_hash).into_string().trim().to_string(); + let model = asset::ActiveModel { + id: Set(id), + nonce: Set(Some(nonce)), + tree_id: Set(Some(tree_id)), + leaf: Set(Some(leaf)), + data_hash: Set(Some(data_hash)), + creator_hash: Set(Some(creator_hash)), + leaf_seq: Set(Some(seq)), + ..Default::default() + }; + + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::column(asset::Column::Id) + .update_columns([ + asset::Column::Nonce, + asset::Column::TreeId, + asset::Column::Leaf, + asset::Column::LeafSeq, + asset::Column::DataHash, + asset::Column::CreatorHash, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + // If we are indexing decompression we will update the leaf regardless of if we have previously + // indexed decompression and regardless of seq. + if !was_decompressed { + query.sql = format!( + "{} WHERE (NOT asset.was_decompressed) AND (excluded.leaf_seq >= asset.leaf_seq OR asset.leaf_seq IS NULL)", + query.sql + ); + } + + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} + +pub async fn upsert_asset_with_leaf_info_for_decompression( + txn: &T, + id: Vec, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let model = asset::ActiveModel { + id: Set(id), + leaf: Set(None), + nonce: Set(Some(0)), + leaf_seq: Set(None), + data_hash: Set(None), + creator_hash: Set(None), + tree_id: Set(None), + seq: Set(Some(0)), + ..Default::default() + }; + let query = asset::Entity::insert(model) + .on_conflict( + OnConflict::column(asset::Column::Id) + .update_columns([ + asset::Column::Leaf, + asset::Column::LeafSeq, + asset::Column::Nonce, + asset::Column::DataHash, + asset::Column::CreatorHash, + asset::Column::TreeId, + asset::Column::Seq, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} + +pub async fn upsert_asset_with_owner_and_delegate_info( + txn: &T, + id: Vec, + owner: Vec, + delegate: Option>, + seq: i64, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let model = asset::ActiveModel { + id: Set(id), + owner: Set(Some(owner)), + delegate: Set(delegate), + owner_delegate_seq: Set(Some(seq)), // gummyroll seq + ..Default::default() + }; + + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::column(asset::Column::Id) + .update_columns([ + asset::Column::Owner, + asset::Column::Delegate, + asset::Column::OwnerDelegateSeq, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.owner_delegate_seq >= asset.owner_delegate_seq OR asset.owner_delegate_seq IS NULL", + query.sql + ); + + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} + +pub async fn upsert_asset_with_compression_info( + txn: &T, + id: Vec, + compressed: bool, + compressible: bool, + supply: i64, + supply_mint: Option>, + was_decompressed: bool, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let model = asset::ActiveModel { + id: Set(id), + compressed: Set(compressed), + compressible: Set(compressible), + supply: Set(supply), + supply_mint: Set(supply_mint), + was_decompressed: Set(was_decompressed), + ..Default::default() + }; + + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::columns([asset::Column::Id]) + .update_columns([ + asset::Column::Compressed, + asset::Column::Compressible, + asset::Column::Supply, + asset::Column::SupplyMint, + asset::Column::WasDecompressed, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!("{} WHERE NOT asset.was_decompressed", query.sql); + txn.execute(query).await?; + + Ok(()) +} + +pub async fn upsert_asset_with_seq(txn: &T, id: Vec, seq: i64) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let model = asset::ActiveModel { + id: Set(id), + seq: Set(Some(seq)), + ..Default::default() + }; + + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::column(asset::Column::Id) + .update_columns([asset::Column::Seq]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + query.sql = format!( + "{} WHERE (NOT asset.was_decompressed) AND (excluded.seq >= asset.seq OR asset.seq IS NULL)", + query.sql + ); + + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} + +pub async fn upsert_creator_verified( + txn: &T, + asset_id: Vec, + creator: Vec, + verified: bool, + seq: i64, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let model = asset_creators::ActiveModel { + asset_id: Set(asset_id), + creator: Set(creator), + verified: Set(verified), + seq: Set(Some(seq)), + ..Default::default() + }; + + let mut query = asset_creators::Entity::insert(model) + .on_conflict( + OnConflict::columns([ + asset_creators::Column::AssetId, + asset_creators::Column::Creator, + ]) + .update_columns([ + asset_creators::Column::Verified, + asset_creators::Column::Seq, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + query.sql = format!( + "{} WHERE excluded.seq >= asset_creators.seq OR asset_creators.seq is NULL", + query.sql + ); + + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} + +pub async fn upsert_collection_info( + txn: &T, + asset_id: Vec, + collection: Option, + slot_updated: i64, + seq: i64, +) -> Result<(), IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + let (group_value, verified) = match collection { + Some(c) => (Some(c.key.to_string()), c.verified), + None => (None, false), + }; + + let model = asset_grouping::ActiveModel { + asset_id: Set(asset_id), + group_key: Set("collection".to_string()), + group_value: Set(group_value), + verified: Set(Some(verified)), + slot_updated: Set(Some(slot_updated)), + group_info_seq: Set(Some(seq)), + ..Default::default() + }; + + let mut query = asset_grouping::Entity::insert(model) + .on_conflict( + OnConflict::columns([ + asset_grouping::Column::AssetId, + asset_grouping::Column::GroupKey, + ]) + .update_columns([ + asset_grouping::Column::GroupValue, + asset_grouping::Column::Verified, + asset_grouping::Column::SlotUpdated, + asset_grouping::Column::GroupInfoSeq, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + query.sql = format!( + "{} WHERE excluded.group_info_seq >= asset_grouping.group_info_seq OR asset_grouping.group_info_seq IS NULL", + query.sql + ); + + txn.execute(query) + .await + .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; + + Ok(()) +} diff --git a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs new file mode 100644 index 000000000..685649f79 --- /dev/null +++ b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs @@ -0,0 +1,364 @@ +use crate::{ + error::IngesterError, + program_transformers::bubblegum::{ + save_changelog_event, upsert_asset_with_compression_info, upsert_asset_with_leaf_info, + upsert_asset_with_owner_and_delegate_info, upsert_asset_with_seq, upsert_collection_info, + }, + tasks::{DownloadMetadata, IntoTaskData, TaskData}, +}; +use blockbuster::{ + instruction::InstructionBundle, + programs::bubblegum::{BubblegumInstruction, LeafSchema, Payload}, + token_metadata::{ + pda::find_master_edition_account, + state::{TokenStandard, UseMethod, Uses}, + }, +}; +use chrono::Utc; +use digital_asset_types::{ + dao::{ + asset, asset_authority, asset_creators, asset_data, asset_v1_account_attachments, + sea_orm_active_enums::{ChainMutability, Mutability, OwnerType, RoyaltyTargetType}, + }, + json::ChainDataV1, +}; +use log::warn; +use num_traits::FromPrimitive; +use sea_orm::{ + entity::*, query::*, sea_query::OnConflict, ConnectionTrait, DbBackend, EntityTrait, JsonValue, +}; +use std::collections::HashSet; + +use digital_asset_types::dao::sea_orm_active_enums::{ + SpecificationAssetClass, SpecificationVersions, V1AccountAttachments, +}; + +// TODO -> consider moving structs into these functions to avoid clone + +pub async fn mint_v1<'c, T>( + parsing_result: &BubblegumInstruction, + bundle: &InstructionBundle<'c>, + txn: &'c T, + cl_audits: bool, +) -> Result, IngesterError> +where + T: ConnectionTrait + TransactionTrait, +{ + if let (Some(le), Some(cl), Some(Payload::MintV1 { args })) = ( + &parsing_result.leaf_update, + &parsing_result.tree_update, + &parsing_result.payload, + ) { + let seq = save_changelog_event(cl, bundle.slot, bundle.txn_id, txn, cl_audits).await?; + let metadata = args; + #[allow(unreachable_patterns)] + return match le.schema { + LeafSchema::V1 { + id, + delegate, + owner, + nonce, + .. + } => { + let (edition_attachment_address, _) = find_master_edition_account(&id); + let id_bytes = id.to_bytes(); + let slot_i = bundle.slot as i64; + let uri = metadata.uri.replace('\0', ""); + let name = metadata.name.clone().into_bytes(); + let symbol = metadata.symbol.clone().into_bytes(); + let mut chain_data = ChainDataV1 { + name: metadata.name.clone(), + symbol: metadata.symbol.clone(), + edition_nonce: metadata.edition_nonce, + primary_sale_happened: metadata.primary_sale_happened, + token_standard: Some(TokenStandard::NonFungible), + uses: metadata.uses.clone().map(|u| Uses { + use_method: UseMethod::from_u8(u.use_method as u8).unwrap(), + remaining: u.remaining, + total: u.total, + }), + }; + chain_data.sanitize(); + let chain_data_json = serde_json::to_value(chain_data) + .map_err(|e| IngesterError::DeserializationError(e.to_string()))?; + let chain_mutability = match metadata.is_mutable { + true => ChainMutability::Mutable, + false => ChainMutability::Immutable, + }; + + let data = asset_data::ActiveModel { + id: Set(id_bytes.to_vec()), + chain_data_mutability: Set(chain_mutability), + chain_data: Set(chain_data_json), + metadata_url: Set(uri.clone()), + metadata: Set(JsonValue::String("processing".to_string())), + metadata_mutability: Set(Mutability::Mutable), + slot_updated: Set(slot_i), + reindex: Set(Some(true)), + raw_name: Set(name.to_vec()), + raw_symbol: Set(symbol.to_vec()), + }; + + let mut query = asset_data::Entity::insert(data) + .on_conflict( + OnConflict::columns([asset_data::Column::Id]) + .update_columns([ + asset_data::Column::ChainDataMutability, + asset_data::Column::ChainData, + asset_data::Column::MetadataUrl, + asset_data::Column::MetadataMutability, + asset_data::Column::SlotUpdated, + asset_data::Column::Reindex, + asset_data::Column::RawName, + asset_data::Column::RawSymbol, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset_data.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + // Insert into `asset` table. + let delegate = if owner == delegate || delegate.to_bytes() == [0; 32] { + None + } else { + Some(delegate.to_bytes().to_vec()) + }; + let tree_id = bundle.keys.get(3).unwrap().0.to_vec(); + + // Set initial mint info. + let asset_model = asset::ActiveModel { + id: Set(id_bytes.to_vec()), + owner_type: Set(OwnerType::Single), + frozen: Set(false), + tree_id: Set(Some(tree_id.clone())), + specification_version: Set(Some(SpecificationVersions::V1)), + specification_asset_class: Set(Some(SpecificationAssetClass::Nft)), + nonce: Set(Some(nonce as i64)), + royalty_target_type: Set(RoyaltyTargetType::Creators), + royalty_target: Set(None), + royalty_amount: Set(metadata.seller_fee_basis_points as i32), //basis points + asset_data: Set(Some(id_bytes.to_vec())), + slot_updated: Set(Some(slot_i)), + ..Default::default() + }; + + // Upsert asset table base info. + let mut query = asset::Entity::insert(asset_model) + .on_conflict( + OnConflict::columns([asset::Column::Id]) + .update_columns([ + asset::Column::OwnerType, + asset::Column::Frozen, + asset::Column::SpecificationVersion, + asset::Column::SpecificationAssetClass, + asset::Column::RoyaltyTargetType, + asset::Column::RoyaltyTarget, + asset::Column::RoyaltyAmount, + asset::Column::AssetData, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + // Do not overwrite changes that happened after the asset was decompressed. + query.sql = format!( + "{} WHERE excluded.slot_updated > asset.slot_updated OR asset.slot_updated IS NULL", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + + // Partial update of asset table with just compression info elements. + upsert_asset_with_compression_info( + txn, + id_bytes.to_vec(), + true, + false, + 1, + None, + false, + ) + .await?; + + // Partial update of asset table with just leaf. + upsert_asset_with_leaf_info( + txn, + id_bytes.to_vec(), + nonce as i64, + tree_id, + le.leaf_hash.to_vec(), + le.schema.data_hash(), + le.schema.creator_hash(), + seq as i64, + false, + ) + .await?; + + // Partial update of asset table with just leaf owner and delegate. + upsert_asset_with_owner_and_delegate_info( + txn, + id_bytes.to_vec(), + owner.to_bytes().to_vec(), + delegate, + seq as i64, + ) + .await?; + + upsert_asset_with_seq(txn, id_bytes.to_vec(), seq as i64).await?; + + let attachment = asset_v1_account_attachments::ActiveModel { + id: Set(edition_attachment_address.to_bytes().to_vec()), + slot_updated: Set(slot_i), + attachment_type: Set(V1AccountAttachments::MasterEditionV2), + ..Default::default() + }; + + let query = asset_v1_account_attachments::Entity::insert(attachment) + .on_conflict( + OnConflict::columns([asset_v1_account_attachments::Column::Id]) + .do_nothing() + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + + // Insert into `asset_creators` table. + let creators = &metadata.creators; + if !creators.is_empty() { + // Vec to hold base creator information. + let mut db_creator_infos = Vec::with_capacity(creators.len()); + + // Vec to hold info on whether a creator is verified. This info is protected by `seq` number. + let mut db_creator_verified_infos = Vec::with_capacity(creators.len()); + + // Set to prevent duplicates. + let mut creators_set = HashSet::new(); + + for (i, c) in creators.iter().enumerate() { + if creators_set.contains(&c.address) { + continue; + } + db_creator_infos.push(asset_creators::ActiveModel { + asset_id: Set(id_bytes.to_vec()), + creator: Set(c.address.to_bytes().to_vec()), + position: Set(i as i16), + share: Set(c.share as i32), + slot_updated: Set(Some(slot_i)), + ..Default::default() + }); + + db_creator_verified_infos.push(asset_creators::ActiveModel { + asset_id: Set(id_bytes.to_vec()), + creator: Set(c.address.to_bytes().to_vec()), + verified: Set(c.verified), + seq: Set(Some(seq as i64)), + ..Default::default() + }); + + creators_set.insert(c.address); + } + + // This statement will update base information for each creator. + let query = asset_creators::Entity::insert_many(db_creator_infos) + .on_conflict( + OnConflict::columns([ + asset_creators::Column::AssetId, + asset_creators::Column::Creator, + ]) + .update_columns([ + asset_creators::Column::Position, + asset_creators::Column::Share, + asset_creators::Column::SlotUpdated, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query).await?; + + // This statement will update whether the creator is verified and the `seq` + // number. `seq` is used to protect the `verified` field, allowing for `mint` + // and `verifyCreator` to be processed out of order. + let mut query = asset_creators::Entity::insert_many(db_creator_verified_infos) + .on_conflict( + OnConflict::columns([ + asset_creators::Column::AssetId, + asset_creators::Column::Creator, + ]) + .update_columns([ + asset_creators::Column::Verified, + asset_creators::Column::Seq, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.seq > asset_creators.seq OR asset_creators.seq IS NULL", + query.sql + ); + txn.execute(query).await?; + } + + // Insert into `asset_authority` table. + let model = asset_authority::ActiveModel { + asset_id: Set(id_bytes.to_vec()), + authority: Set(bundle.keys.get(0).unwrap().0.to_vec()), //TODO - we need to rem,ove the optional bubblegum signer logic + seq: Set(seq as i64), + slot_updated: Set(slot_i), + ..Default::default() + }; + + // Do not attempt to modify any existing values: + // `ON CONFLICT ('asset_id') DO NOTHING`. + let query = asset_authority::Entity::insert(model) + .on_conflict( + OnConflict::columns([asset_authority::Column::AssetId]) + .do_nothing() + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + + // Upsert into `asset_grouping` table with base collection info. + upsert_collection_info( + txn, + id_bytes.to_vec(), + metadata.collection.clone(), + slot_i, + seq as i64, + ) + .await?; + + if uri.is_empty() { + warn!( + "URI is empty for mint {}. Skipping background task.", + bs58::encode(id).into_string() + ); + return Ok(None); + } + + let mut task = DownloadMetadata { + asset_data_id: id_bytes.to_vec(), + uri: metadata.uri.clone(), + created_at: Some(Utc::now().naive_utc()), + }; + task.sanitize(); + let t = task.into_task_data()?; + Ok(Some(t)) + } + _ => Err(IngesterError::NotImplemented), + }; + } + Err(IngesterError::ParsingError( + "Ix not parsed correctly".to_string(), + )) +} diff --git a/nft_ingester/src/program_transformers/mod.rs b/nft_ingester/src/program_transformers/mod.rs new file mode 100644 index 000000000..fc555829d --- /dev/null +++ b/nft_ingester/src/program_transformers/mod.rs @@ -0,0 +1,189 @@ +use crate::{error::IngesterError, tasks::TaskData}; +use blockbuster::{ + instruction::{order_instructions, InstructionBundle, IxPair}, + program_handler::ProgramParser, + programs::{ + bubblegum::BubblegumParser, token_account::TokenAccountParser, + token_metadata::TokenMetadataParser, ProgramParseResult, + }, +}; +use log::{debug, error, info}; +use plerkle_serialization::{AccountInfo, Pubkey as FBPubkey, TransactionInfo}; +use sea_orm::{DatabaseConnection, SqlxPostgresConnector}; +use solana_sdk::pubkey::Pubkey; +use sqlx::PgPool; +use std::collections::{HashMap, HashSet, VecDeque}; +use tokio::sync::mpsc::UnboundedSender; + +use crate::program_transformers::{ + bubblegum::handle_bubblegum_instruction, token::handle_token_program_account, + token_metadata::handle_token_metadata_account, +}; + +mod bubblegum; +mod token; +mod token_metadata; + +pub struct ProgramTransformer { + storage: DatabaseConnection, + task_sender: UnboundedSender, + matchers: HashMap>, + key_set: HashSet, + cl_audits: bool, +} + +impl ProgramTransformer { + pub fn new(pool: PgPool, task_sender: UnboundedSender, cl_audits: bool) -> Self { + let mut matchers: HashMap> = HashMap::with_capacity(1); + let bgum = BubblegumParser {}; + let token_metadata = TokenMetadataParser {}; + let token = TokenAccountParser {}; + matchers.insert(bgum.key(), Box::new(bgum)); + matchers.insert(token_metadata.key(), Box::new(token_metadata)); + matchers.insert(token.key(), Box::new(token)); + let hs = matchers.iter().fold(HashSet::new(), |mut acc, (k, _)| { + acc.insert(*k); + acc + }); + let pool: PgPool = pool; + ProgramTransformer { + storage: SqlxPostgresConnector::from_sqlx_postgres_pool(pool), + task_sender, + matchers, + key_set: hs, + cl_audits, + } + } + + pub fn break_transaction<'i>( + &self, + tx: &'i TransactionInfo<'i>, + ) -> VecDeque<(IxPair<'i>, Option>>)> { + let ref_set: HashSet<&[u8]> = self.key_set.iter().map(|k| k.as_ref()).collect(); + order_instructions(ref_set, tx) + } + + #[allow(clippy::borrowed_box)] + pub fn match_program(&self, key: &FBPubkey) -> Option<&Box> { + self.matchers + .get(&Pubkey::try_from(key.0.as_slice()).expect("valid key from FlatBuffer")) + } + + pub async fn handle_transaction<'a>( + &self, + tx: &'a TransactionInfo<'a>, + ) -> Result<(), IngesterError> { + let sig: Option<&str> = tx.signature(); + info!("Handling Transaction: {:?}", sig); + let instructions = self.break_transaction(tx); + let accounts = tx.account_keys().unwrap_or_default(); + let slot = tx.slot(); + let txn_id = tx.signature().unwrap_or(""); + let mut keys: Vec = Vec::with_capacity(accounts.len()); + for k in accounts.into_iter() { + keys.push(*k); + } + let mut not_impl = 0; + let ixlen = instructions.len(); + debug!("Instructions: {}", ixlen); + let contains = instructions + .iter() + .filter(|(ib, _inner)| ib.0 .0.as_ref() == mpl_bubblegum::ID.as_ref()); + debug!("Instructions bgum: {}", contains.count()); + for (outer_ix, inner_ix) in instructions { + let (program, instruction) = outer_ix; + let ix_accounts = instruction.accounts().unwrap().iter().collect::>(); + let ix_account_len = ix_accounts.len(); + let max = ix_accounts.iter().max().copied().unwrap_or(0) as usize; + if keys.len() < max { + return Err(IngesterError::DeserializationError( + "Missing Accounts in Serialized Ixn/Txn".to_string(), + )); + } + let ix_accounts = + ix_accounts + .iter() + .fold(Vec::with_capacity(ix_account_len), |mut acc, a| { + if let Some(key) = keys.get(*a as usize) { + acc.push(*key); + } + acc + }); + let ix = InstructionBundle { + txn_id, + program, + instruction: Some(instruction), + inner_ix, + keys: ix_accounts.as_slice(), + slot, + }; + + if let Some(program) = self.match_program(&ix.program) { + debug!("Found a ix for program: {:?}", program.key()); + let result = program.handle_instruction(&ix)?; + let concrete = result.result_type(); + match concrete { + ProgramParseResult::Bubblegum(parsing_result) => { + handle_bubblegum_instruction( + parsing_result, + &ix, + &self.storage, + &self.task_sender, + self.cl_audits, + ) + .await + .map_err(|err| { + error!( + "Failed to handle bubblegum instruction for txn {:?}: {:?}", + sig, err + ); + err + })?; + } + _ => { + not_impl += 1; + } + }; + } + } + + if not_impl == ixlen { + debug!("Not imple"); + return Err(IngesterError::NotImplemented); + } + Ok(()) + } + + pub async fn handle_account_update<'b>( + &self, + acct: AccountInfo<'b>, + ) -> Result<(), IngesterError> { + let owner = acct.owner().unwrap(); + if let Some(program) = self.match_program(owner) { + let result = program.handle_account(&acct)?; + let concrete = result.result_type(); + match concrete { + ProgramParseResult::TokenMetadata(parsing_result) => { + handle_token_metadata_account( + &acct, + parsing_result, + &self.storage, + &self.task_sender, + ) + .await + } + ProgramParseResult::TokenProgramAccount(parsing_result) => { + handle_token_program_account( + &acct, + parsing_result, + &self.storage, + &self.task_sender, + ) + .await + } + _ => Err(IngesterError::NotImplemented), + }?; + } + Ok(()) + } +} diff --git a/nft_ingester/src/program_transformers/token/mod.rs b/nft_ingester/src/program_transformers/token/mod.rs new file mode 100644 index 000000000..c32cad020 --- /dev/null +++ b/nft_ingester/src/program_transformers/token/mod.rs @@ -0,0 +1,141 @@ +use crate::{error::IngesterError, tasks::TaskData}; +use blockbuster::programs::token_account::TokenProgramAccount; +use digital_asset_types::dao::{asset, token_accounts, tokens}; +use plerkle_serialization::AccountInfo; +use sea_orm::{ + entity::*, query::*, sea_query::OnConflict, ActiveValue::Set, ConnectionTrait, + DatabaseConnection, DbBackend, EntityTrait, +}; +use solana_sdk::program_option::COption; +use spl_token::state::AccountState; +use tokio::sync::mpsc::UnboundedSender; + +pub async fn handle_token_program_account<'a, 'b, 'c>( + account_update: &'a AccountInfo<'a>, + parsing_result: &'b TokenProgramAccount, + db: &'c DatabaseConnection, + _task_manager: &UnboundedSender, +) -> Result<(), IngesterError> { + let key = *account_update.pubkey().unwrap(); + let key_bytes = key.0.to_vec(); + let spl_token_program = account_update.owner().unwrap().0.to_vec(); + match &parsing_result { + TokenProgramAccount::TokenAccount(ta) => { + let mint = ta.mint.to_bytes().to_vec(); + let delegate: Option> = match ta.delegate { + COption::Some(d) => Some(d.to_bytes().to_vec()), + COption::None => None, + }; + let frozen = matches!(ta.state, AccountState::Frozen); + let owner = ta.owner.to_bytes().to_vec(); + let model = token_accounts::ActiveModel { + pubkey: Set(key_bytes), + mint: Set(mint.clone()), + delegate: Set(delegate.clone()), + owner: Set(owner.clone()), + frozen: Set(frozen), + delegated_amount: Set(ta.delegated_amount as i64), + token_program: Set(spl_token_program), + slot_updated: Set(account_update.slot() as i64), + amount: Set(ta.amount as i64), + close_authority: Set(None), + }; + + let mut query = token_accounts::Entity::insert(model) + .on_conflict( + OnConflict::columns([token_accounts::Column::Pubkey]) + .update_columns([ + token_accounts::Column::Mint, + token_accounts::Column::DelegatedAmount, + token_accounts::Column::Delegate, + token_accounts::Column::Amount, + token_accounts::Column::Frozen, + token_accounts::Column::TokenProgram, + token_accounts::Column::Owner, + token_accounts::Column::CloseAuthority, + token_accounts::Column::SlotUpdated, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > token_accounts.slot_updated", + query.sql + ); + db.execute(query).await?; + let txn = db.begin().await?; + let asset_update: Option = asset::Entity::find_by_id(mint) + .filter(asset::Column::OwnerType.eq("single")) + .one(&txn) + .await?; + if let Some(asset) = asset_update { + // will only update owner if token account balance is non-zero + if ta.amount > 0 { + let mut active: asset::ActiveModel = asset.into(); + active.owner = Set(Some(owner)); + active.delegate = Set(delegate); + active.frozen = Set(frozen); + active.save(&txn).await?; + } + } + txn.commit().await?; + Ok(()) + } + TokenProgramAccount::Mint(m) => { + let freeze_auth: Option> = match m.freeze_authority { + COption::Some(d) => Some(d.to_bytes().to_vec()), + COption::None => None, + }; + let mint_auth: Option> = match m.mint_authority { + COption::Some(d) => Some(d.to_bytes().to_vec()), + COption::None => None, + }; + let model = tokens::ActiveModel { + mint: Set(key_bytes.clone()), + token_program: Set(spl_token_program), + slot_updated: Set(account_update.slot() as i64), + supply: Set(m.supply as i64), + decimals: Set(m.decimals as i32), + close_authority: Set(None), + extension_data: Set(None), + mint_authority: Set(mint_auth), + freeze_authority: Set(freeze_auth), + }; + + let mut query = tokens::Entity::insert(model) + .on_conflict( + OnConflict::columns([tokens::Column::Mint]) + .update_columns([ + tokens::Column::Supply, + tokens::Column::TokenProgram, + tokens::Column::MintAuthority, + tokens::Column::CloseAuthority, + tokens::Column::ExtensionData, + tokens::Column::SlotUpdated, + tokens::Column::Decimals, + tokens::Column::FreezeAuthority, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > tokens.slot_updated", + query.sql + ); + db.execute(query).await?; + let asset_update: Option = asset::Entity::find_by_id(key_bytes.clone()) + .filter(asset::Column::OwnerType.eq("single")) + .one(db) + .await?; + if let Some(asset) = asset_update { + let mut active: asset::ActiveModel = asset.into(); + active.supply = Set(m.supply as i64); + active.supply_mint = Set(Some(key_bytes)); + active.save(db).await?; + } + Ok(()) + } + _ => Err(IngesterError::NotImplemented), + }?; + Ok(()) +} diff --git a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs new file mode 100644 index 000000000..4f81a4636 --- /dev/null +++ b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs @@ -0,0 +1,393 @@ +use crate::{error::IngesterError, tasks::TaskData}; +use blockbuster::token_metadata::{ + pda::find_master_edition_account, + state::{Metadata, TokenStandard, UseMethod, Uses}, +}; +use chrono::Utc; +use digital_asset_types::{ + dao::{ + asset, asset_authority, asset_creators, asset_data, asset_grouping, + asset_v1_account_attachments, + sea_orm_active_enums::{ + ChainMutability, Mutability, OwnerType, RoyaltyTargetType, SpecificationAssetClass, + SpecificationVersions, V1AccountAttachments, + }, + token_accounts, tokens, + }, + json::ChainDataV1, +}; + +use crate::tasks::{DownloadMetadata, IntoTaskData}; +use log::warn; +use num_traits::FromPrimitive; +use plerkle_serialization::Pubkey as FBPubkey; +use sea_orm::{ + entity::*, query::*, sea_query::OnConflict, ActiveValue::Set, ConnectionTrait, DbBackend, + DbErr, EntityTrait, JsonValue, +}; +use std::collections::HashSet; + +pub async fn burn_v1_asset( + conn: &T, + id: FBPubkey, + slot: u64, +) -> Result<(), IngesterError> { + let (id, slot_i) = (id.0, slot as i64); + let model = asset::ActiveModel { + id: Set(id.to_vec()), + slot_updated: Set(Some(slot_i)), + burnt: Set(true), + ..Default::default() + }; + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::columns([asset::Column::Id]) + .update_columns([asset::Column::SlotUpdated, asset::Column::Burnt]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset.slot_updated", + query.sql + ); + conn.execute(query).await?; + Ok(()) +} + +pub async fn save_v1_asset( + conn: &T, + id: FBPubkey, + slot: u64, + metadata: &Metadata, +) -> Result, IngesterError> { + let metadata = metadata.clone(); + let data = metadata.data; + let meta_mint_pubkey = metadata.mint; + let (edition_attachment_address, _) = find_master_edition_account(&meta_mint_pubkey); + let mint = metadata.mint.to_bytes().to_vec(); + let authority = metadata.update_authority.to_bytes().to_vec(); + let id = id.0; + let slot_i = slot as i64; + let uri = data.uri.trim().replace('\0', ""); + let _spec = SpecificationVersions::V1; + let class = match metadata.token_standard { + Some(TokenStandard::NonFungible) => SpecificationAssetClass::Nft, + Some(TokenStandard::FungibleAsset) => SpecificationAssetClass::FungibleAsset, + Some(TokenStandard::Fungible) => SpecificationAssetClass::FungibleToken, + _ => SpecificationAssetClass::Unknown, + }; + let ownership_type = match class { + SpecificationAssetClass::FungibleAsset => OwnerType::Token, + SpecificationAssetClass::FungibleToken => OwnerType::Token, + _ => OwnerType::Single, + }; + + // gets the token and token account for the mint to populate the asset. This is required when the token and token account are indexed, but not the metadata account. If the metadata account is indexed, then the token and ta ingester will update the asset with the correct data + + let (token, token_account): (Option, Option) = + match ownership_type { + OwnerType::Single => { + let token: Option = + tokens::Entity::find_by_id(mint.clone()).one(conn).await?; + // query for token account associated with mint with positive balance + let token_account: Option = token_accounts::Entity::find() + .filter(token_accounts::Column::Mint.eq(mint.clone())) + .filter(token_accounts::Column::Amount.gt(0)) + .one(conn) + .await?; + Ok((token, token_account)) + } + _ => { + let token = tokens::Entity::find_by_id(mint.clone()).one(conn).await?; + Ok((token, None)) + } + } + .map_err(|e: DbErr| IngesterError::DatabaseError(e.to_string()))?; + + // get supply of token, default to 1 since most cases will be NFTs. Token mint ingester will properly set supply if token_result is None + let (supply, supply_mint) = match token { + Some(t) => (Set(t.supply), Set(Some(t.mint))), + None => (Set(1), NotSet), + }; + + // owner and delegate should be from the token account with the mint + let (owner, delegate) = match token_account { + Some(ta) => (Set(Some(ta.owner)), Set(ta.delegate)), + None => (NotSet, NotSet), + }; + + let name = data.name.clone().into_bytes(); + let symbol = data.symbol.clone().into_bytes(); + + let mut chain_data = ChainDataV1 { + name: data.name.clone(), + symbol: data.symbol.clone(), + edition_nonce: metadata.edition_nonce, + primary_sale_happened: metadata.primary_sale_happened, + token_standard: metadata.token_standard, + uses: metadata.uses.map(|u| Uses { + use_method: UseMethod::from_u8(u.use_method as u8).unwrap(), + remaining: u.remaining, + total: u.total, + }), + }; + chain_data.sanitize(); + let chain_data_json = serde_json::to_value(chain_data) + .map_err(|e| IngesterError::DeserializationError(e.to_string()))?; + let chain_mutability = match metadata.is_mutable { + true => ChainMutability::Mutable, + false => ChainMutability::Immutable, + }; + let asset_data_model = asset_data::ActiveModel { + chain_data_mutability: Set(chain_mutability), + chain_data: Set(chain_data_json), + metadata_url: Set(uri.clone()), + metadata: Set(JsonValue::String("processing".to_string())), + metadata_mutability: Set(Mutability::Mutable), + slot_updated: Set(slot_i), + reindex: Set(Some(true)), + id: Set(id.to_vec()), + raw_name: Set(name.to_vec()), + raw_symbol: Set(symbol.to_vec()), + }; + let txn = conn.begin().await?; + let mut query = asset_data::Entity::insert(asset_data_model) + .on_conflict( + OnConflict::columns([asset_data::Column::Id]) + .update_columns([ + asset_data::Column::ChainDataMutability, + asset_data::Column::ChainData, + asset_data::Column::MetadataUrl, + asset_data::Column::MetadataMutability, + asset_data::Column::SlotUpdated, + asset_data::Column::Reindex, + asset_data::Column::RawName, + asset_data::Column::RawSymbol, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset_data.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + let model = asset::ActiveModel { + id: Set(id.to_vec()), + owner, + owner_type: Set(ownership_type), + delegate, + frozen: Set(false), + supply, + supply_mint, + specification_version: Set(Some(SpecificationVersions::V1)), + specification_asset_class: Set(Some(class)), + tree_id: Set(None), + nonce: Set(Some(0)), + seq: Set(Some(0)), + leaf: Set(None), + compressed: Set(false), + compressible: Set(false), + royalty_target_type: Set(RoyaltyTargetType::Creators), + royalty_target: Set(None), + royalty_amount: Set(data.seller_fee_basis_points as i32), //basis points + asset_data: Set(Some(id.to_vec())), + slot_updated: Set(Some(slot_i)), + burnt: Set(false), + ..Default::default() + }; + let mut query = asset::Entity::insert(model) + .on_conflict( + OnConflict::columns([asset::Column::Id]) + .update_columns([ + asset::Column::Owner, + asset::Column::OwnerType, + asset::Column::Delegate, + asset::Column::Frozen, + asset::Column::Supply, + asset::Column::SupplyMint, + asset::Column::SpecificationVersion, + asset::Column::SpecificationAssetClass, + asset::Column::TreeId, + asset::Column::Nonce, + asset::Column::Seq, + asset::Column::Leaf, + asset::Column::Compressed, + asset::Column::Compressible, + asset::Column::RoyaltyTargetType, + asset::Column::RoyaltyTarget, + asset::Column::RoyaltyAmount, + asset::Column::AssetData, + asset::Column::SlotUpdated, + asset::Column::Burnt, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + let attachment = asset_v1_account_attachments::ActiveModel { + id: Set(edition_attachment_address.to_bytes().to_vec()), + slot_updated: Set(slot_i), + attachment_type: Set(V1AccountAttachments::MasterEditionV2), + ..Default::default() + }; + let query = asset_v1_account_attachments::Entity::insert(attachment) + .on_conflict( + OnConflict::columns([asset_v1_account_attachments::Column::Id]) + .do_nothing() + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + let model = asset_authority::ActiveModel { + asset_id: Set(id.to_vec()), + authority: Set(authority), + seq: Set(0), + slot_updated: Set(slot_i), + ..Default::default() + }; + let mut query = asset_authority::Entity::insert(model) + .on_conflict( + OnConflict::columns([asset_authority::Column::AssetId]) + .update_columns([ + asset_authority::Column::Authority, + asset_authority::Column::Seq, + asset_authority::Column::SlotUpdated, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset_authority.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + if let Some(c) = &metadata.collection { + let model = asset_grouping::ActiveModel { + asset_id: Set(id.to_vec()), + group_key: Set("collection".to_string()), + group_value: Set(Some(c.key.to_string())), + verified: Set(Some(c.verified)), + seq: Set(None), + slot_updated: Set(Some(slot_i)), + ..Default::default() + }; + let mut query = asset_grouping::Entity::insert(model) + .on_conflict( + OnConflict::columns([ + asset_grouping::Column::AssetId, + asset_grouping::Column::GroupKey, + ]) + .update_columns([ + asset_grouping::Column::GroupKey, + asset_grouping::Column::GroupValue, + asset_grouping::Column::Seq, + asset_grouping::Column::SlotUpdated, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset_grouping.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + } + txn.commit().await?; + let creators = data.creators.unwrap_or_default(); + if !creators.is_empty() { + let mut creators_set = HashSet::new(); + let existing_creators: Vec = asset_creators::Entity::find() + .filter( + Condition::all() + .add(asset_creators::Column::AssetId.eq(id.to_vec())) + .add(asset_creators::Column::SlotUpdated.lt(slot_i)), + ) + .all(conn) + .await?; + if !existing_creators.is_empty() { + let mut db_creators = Vec::with_capacity(creators.len()); + for (i, c) in creators.into_iter().enumerate() { + if creators_set.contains(&c.address) { + continue; + } + db_creators.push(asset_creators::ActiveModel { + asset_id: Set(id.to_vec()), + creator: Set(c.address.to_bytes().to_vec()), + share: Set(c.share as i32), + verified: Set(c.verified), + seq: Set(Some(0)), + slot_updated: Set(Some(slot_i)), + position: Set(i as i16), + ..Default::default() + }); + creators_set.insert(c.address); + } + let txn = conn.begin().await?; + asset_creators::Entity::delete_many() + .filter( + Condition::all() + .add(asset_creators::Column::AssetId.eq(id.to_vec())) + .add(asset_creators::Column::SlotUpdated.lt(slot_i)), + ) + .exec(&txn) + .await?; + if !db_creators.is_empty() { + let mut query = asset_creators::Entity::insert_many(db_creators) + .on_conflict( + OnConflict::columns([ + asset_creators::Column::AssetId, + asset_creators::Column::Position, + ]) + .update_columns([ + asset_creators::Column::Creator, + asset_creators::Column::Share, + asset_creators::Column::Verified, + asset_creators::Column::Seq, + asset_creators::Column::SlotUpdated, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + query.sql = format!( + "{} WHERE excluded.slot_updated > asset_creators.slot_updated", + query.sql + ); + txn.execute(query) + .await + .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; + } + txn.commit().await?; + } + } + if uri.is_empty() { + warn!( + "URI is empty for mint {}. Skipping background task.", + bs58::encode(mint).into_string() + ); + return Ok(None); + } + + let mut task = DownloadMetadata { + asset_data_id: id.to_vec(), + uri, + created_at: Some(Utc::now().naive_utc()), + }; + task.sanitize(); + let t = task.into_task_data()?; + Ok(Some(t)) +} From 974c90fe2add57fe26fa571a43d456635d8c62c6 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 20 Nov 2023 12:41:19 -0500 Subject: [PATCH 03/72] ci: add lock, fmt, clippy checks --- .github/workflows/build-api.yml | 32 ++++++++++++++++++++------------ .github/workflows/test.yml | 18 +++++++++--------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index 15684355a..dfe23bce6 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -12,10 +12,10 @@ on: workflow_dispatch: push: branches: - - 'main' + - "main" env: - CARGO_TERM_COLOR: always + CARGO_TERM_COLOR: always jobs: build-api: @@ -33,19 +33,27 @@ jobs: - name: set build cache uses: actions/cache@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - digital-asset-rpc-infrastructure/target/ - key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }} - restore-keys: | - ${{ matrix.os }}_digital-asset-rpc-infrastructure + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + digital-asset-rpc-infrastructure/target/ + key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }} + restore-keys: | + ${{ matrix.os }}_digital-asset-rpc-infrastructure - name: build digital asset rpc infra run: cargo build --verbose --release + - name: build das_api + working-directory: das_api + run: cargo build --verbose --release + + - name: build migration + working-directory: migration + run: cargo build --verbose --release + - name: rename binaries for ubuntu22 release if: matrix.os == 'ubuntu-22.04' run: | @@ -54,7 +62,7 @@ jobs: mv target/release/migration target/release/migration22 mv target/release/das_api target/release/das_api22 - # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them + # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them # Omiting this will save on storage consumption on the account - name: Publish artifact if: matrix.os == 'ubuntu-22.04' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ced2cd37..4f4b4111d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,11 +8,11 @@ on: pull_request: push: branches: - - 'main' + - "main" workflow_dispatch: env: - CARGO_TERM_COLOR: always + CARGO_TERM_COLOR: always jobs: test: @@ -25,13 +25,13 @@ jobs: - name: set build cache uses: actions/cache@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - digital-asset-rpc-infrastructure/target/ - key: cargo-${{ hashFiles('**/Cargo.lock') }}-0001 + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + digital-asset-rpc-infrastructure/target/ + key: cargo-${{ hashFiles('**/Cargo.lock') }}-0001 # Cargo.lock - name: Check lock file From 788973e2724a50405a76b68127ec6ba936299bdc Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 20 Nov 2023 12:57:28 -0500 Subject: [PATCH 04/72] include migration to workspace --- .github/workflows/build-api.yml | 4 - Cargo.lock | 1681 +++++++++-------- ...131_140613_change_token_account_indexes.rs | 11 + 3 files changed, 926 insertions(+), 770 deletions(-) diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index dfe23bce6..8082f52a9 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -50,10 +50,6 @@ jobs: working-directory: das_api run: cargo build --verbose --release - - name: build migration - working-directory: migration - run: cargo build --verbose --release - - name: rename binaries for ubuntu22 release if: matrix.os == 'ubuntu-22.04' run: | diff --git a/Cargo.lock b/Cargo.lock index 90e706291..8a5e4dd91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,8 +18,8 @@ version = "0.7.12" dependencies = [ "anyhow", "bs58 0.4.0", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.4", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -93,11 +93,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", "once_cell", "version_check", ] @@ -109,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.14", "once_cell", "version_check", "zerocopy", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f2a3e1df4685f18d12a943a9f2a7456305401af21a07c9fe076ef9ecd6e400" dependencies = [ "anchor-syn", - "bs58 0.5.0", + "bs58 0.5.1", "proc-macro2", "quote", "syn 1.0.109", @@ -295,7 +295,7 @@ dependencies = [ "bincode", "borsh 0.10.3", "bytemuck", - "getrandom 0.2.10", + "getrandom 0.2.14", "solana-program", "thiserror", ] @@ -307,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9101b84702fed2ea57bd22992f75065da5648017135b844283a2f6d74f27825" dependencies = [ "anyhow", - "bs58 0.5.0", + "bs58 0.5.1", "heck 0.3.3", "proc-macro2", "quote", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -358,49 +358,49 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "ark-bn254" @@ -550,7 +550,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.29", + "time 0.3.35", ] [[package]] @@ -605,22 +605,22 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener 3.1.0", - "event-listener-strategy", + "event-listener 5.3.0", + "event-listener-strategy 0.5.1", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" dependencies = [ "brotli", "flate2", @@ -632,30 +632,30 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.7.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" +checksum = "5f98c37cf288e302c16ef6c8472aad1e034c6c84ce5ea7b8101c98eb4a802fee" dependencies = [ - "async-lock 3.1.1", + "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.0.1", + "fastrand 2.0.2", + "futures-lite 2.3.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 1.9.0", + "async-channel 2.2.0", "async-executor", - "async-io", - "async-lock 2.8.0", + "async-io 2.3.2", + "async-lock 3.3.0", "blocking", - "futures-lite 1.13.0", + "futures-lite 2.3.0", "once_cell", "tokio", ] @@ -673,13 +673,32 @@ dependencies = [ "futures-lite 1.13.0", "log", "parking", - "polling", + "polling 2.8.0", "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.3.0", + "parking", + "polling 3.6.0", + "rustix 0.38.32", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" @@ -691,12 +710,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.1.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ - "event-listener 3.1.0", - "event-listener-strategy", + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", "pin-project-lite", ] @@ -718,7 +737,7 @@ dependencies = [ "async-attributes", "async-channel 1.9.0", "async-global-executor", - "async-io", + "async-io 1.13.0", "async-lock 2.8.0", "crossbeam-utils", "futures-channel", @@ -755,24 +774,24 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "async-task" -version = "4.5.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -818,17 +837,17 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "backon" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ - "fastrand 1.9.0", + "fastrand 2.0.2", "futures-core", "pin-project", "tokio", @@ -836,9 +855,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -876,9 +895,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -906,7 +925,7 @@ name = "bgtask_creator" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", "futures", "lazy_static", @@ -938,9 +957,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" dependencies = [ "serde", ] @@ -968,9 +987,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", @@ -1041,12 +1060,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.1.0", - "async-lock 3.1.1", + "async-channel 2.2.0", + "async-lock 3.3.0", "async-task", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", - "futures-lite 2.0.1", + "futures-lite 2.3.0", "piper", "tracing", ] @@ -1071,6 +1090,16 @@ dependencies = [ "hashbrown 0.13.2", ] +[[package]] +name = "borsh" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0901fc8eb0aca4c83be0106d6f2db17d86a08dfc2c25f0e84464bf381158add6" +dependencies = [ + "borsh-derive 1.4.0", + "cfg_aliases", +] + [[package]] name = "borsh-derive" version = "0.9.3" @@ -1097,6 +1126,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51670c3aa053938b0ee3bd67c3817e471e626151131b934038e83c5bf8de48f5" +dependencies = [ + "once_cell", + "proc-macro-crate 3.1.0", + "proc-macro2", + "quote", + "syn 2.0.58", + "syn_derive", +] + [[package]] name = "borsh-derive-internal" version = "0.9.3" @@ -1143,9 +1186,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1154,9 +1197,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" +checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1170,18 +1213,18 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", ] [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "serde", @@ -1189,9 +1232,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bv" @@ -1205,9 +1248,9 @@ dependencies = [ [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1216,9 +1259,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", @@ -1236,13 +1279,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1253,9 +1296,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cadence" @@ -1287,9 +1330,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" dependencies = [ "jobserver", "libc", @@ -1301,11 +1344,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1313,7 +1362,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.4", ] [[package]] @@ -1354,29 +1403,29 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.0", + "textwrap 0.16.1", ] [[package]] name = "clap" -version = "4.4.8" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", - "clap_derive 4.4.7", + "clap_derive 4.5.4", ] [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", - "strsim 0.10.0", + "clap_lex 0.7.0", + "strsim 0.11.1", ] [[package]] @@ -1394,14 +1443,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1415,9 +1464,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -1440,9 +1489,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", @@ -1454,24 +1503,24 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -1508,9 +1557,9 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1518,95 +1567,85 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1659,9 +1698,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ "darling_core", "darling_macro", @@ -1669,27 +1708,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1700,7 +1739,7 @@ dependencies = [ "backon", "cadence", "cadence-macros", - "clap 4.4.8", + "clap 4.5.4", "figment", "plerkle_messenger", "solana-account-decoder", @@ -1722,10 +1761,10 @@ dependencies = [ "borsh 0.10.3", "cadence", "cadence-macros", - "clap 4.4.8", + "clap 4.5.4", "das-core", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -1755,7 +1794,7 @@ dependencies = [ "cadence", "cadence-macros", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "figment", "hyper", "jsonrpsee", @@ -1797,17 +1836,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.9", ] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "der" @@ -1834,10 +1873,11 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -1953,7 +1993,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1976,7 +2016,7 @@ checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1987,9 +2027,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "eager" @@ -2034,9 +2074,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" dependencies = [ "serde", ] @@ -2049,31 +2089,31 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -2091,9 +2131,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -2110,12 +2150,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2126,9 +2166,20 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "3.1.0" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" dependencies = [ "concurrent-queue", "parking", @@ -2137,11 +2188,21 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.3.0" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" +checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" dependencies = [ - "event-listener 3.1.0", + "event-listener 5.3.0", "pin-project-lite", ] @@ -2166,9 +2227,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "feature-probe" @@ -2183,7 +2244,7 @@ dependencies = [ "anyhow", "async-trait", "borsh 0.10.3", - "clap 4.4.8", + "clap 4.5.4", "mpl-bubblegum", "solana-account-decoder", "solana-client", @@ -2194,15 +2255,15 @@ dependencies = [ [[package]] name = "figment" -version = "0.10.11" +version = "0.10.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a014ac935975a70ad13a3bff2463b1c1b083b35ae4cb6309cfc59476aa7a181f" +checksum = "fdefe49ed1057d124dc81a0681c30dd07de56ad96e32adc7b64e8f28eaab31c4" dependencies = [ "atomic 0.6.0", "pear", "serde", "serde_yaml", - "toml 0.8.8", + "toml 0.8.12", "uncased", "version_check", ] @@ -2225,9 +2286,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -2256,9 +2317,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -2292,9 +2353,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2307,9 +2368,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2317,15 +2378,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2345,9 +2406,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2366,46 +2427,45 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-core", "futures-io", - "memchr", "parking", "pin-project-lite", ] [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2461,9 +2521,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -2474,21 +2534,21 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -2516,9 +2576,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -2526,7 +2586,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -2548,7 +2608,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2557,7 +2617,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2571,9 +2631,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash 0.8.5", "allocator-api2", @@ -2585,7 +2645,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -2623,9 +2683,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2641,9 +2701,9 @@ checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac 0.12.1", ] @@ -2680,9 +2740,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2691,9 +2751,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2726,9 +2786,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2741,7 +2801,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2750,14 +2810,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", ] @@ -2777,16 +2837,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -2806,9 +2866,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2843,25 +2903,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "serde", ] [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", "number_prefix", - "portable-atomic 1.4.3", + "portable-atomic 1.6.0", "unicode-width", ] @@ -2873,16 +2933,15 @@ checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "insta" -version = "1.35.1" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" +checksum = "3eab73f58e59ca6526037208f0e98851159ec1633cf17b6cd2e1f2c3fd5d53cc" dependencies = [ "console", "lazy_static", "linked-hash-map", "serde", "similar", - "yaml-rust", ] [[package]] @@ -2934,26 +2993,26 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.18", - "windows-sys 0.48.0", + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -2967,33 +3026,33 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -3114,18 +3173,18 @@ dependencies = [ [[package]] name = "kaigan" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a26f49495f94a283312e7ef45a243540ef20c9356bb01c8d84a61ac8ba5339b" +checksum = "e623cca1f0e2a0919032c1bdabbf81dd9aa34658d5066aca7bb90d608317ab91" dependencies = [ "borsh 0.10.3", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] @@ -3147,9 +3206,19 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] [[package]] name = "libsecp256k1" @@ -3225,9 +3294,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "load_generation" @@ -3246,9 +3315,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -3256,9 +3325,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" dependencies = [ "value-bag", ] @@ -3284,9 +3353,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memmap2" @@ -3308,9 +3377,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -3345,7 +3414,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "metrics-macros", "portable-atomic 0.3.20", ] @@ -3395,18 +3464,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3438,22 +3507,22 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", + "serde_with 3.7.0", "solana-program", "thiserror", ] [[package]] name = "mpl-token-metadata" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b2de608098eb2ef2a5392069dea83084967e25a4d69d0380a6bb02454fc0fe" +checksum = "caf0f61b553e424a6234af1268456972ee66c2222e1da89079242251fa7479e5" dependencies = [ "borsh 0.10.3", "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", + "serde_with 3.7.0", "solana-program", "thiserror", ] @@ -3487,7 +3556,7 @@ dependencies = [ "cadence", "cadence-macros", "chrono", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", "figment", "flatbuffers", @@ -3595,6 +3664,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -3608,30 +3683,29 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -3652,9 +3726,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -3665,7 +3739,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", ] @@ -3696,7 +3770,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3705,10 +3779,10 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3719,9 +3793,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -3743,9 +3817,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open-rpc-derive" @@ -3773,11 +3847,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -3794,7 +3868,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3805,9 +3879,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -3874,7 +3948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.9", ] [[package]] @@ -3893,13 +3967,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -3930,9 +4004,9 @@ dependencies = [ [[package]] name = "pear" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a386cd715229d399604b50d1361683fe687066f42d56f54be995bc6868f71c" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", @@ -3941,14 +4015,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f0f13dac8069c139e8300a6510e3f4143ecf5259c60b116a9b271b4ca0d54" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3962,9 +4036,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "percentage" @@ -3977,29 +4051,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -4014,7 +4088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", ] @@ -4031,9 +4105,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plain" @@ -4043,9 +4117,9 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plerkle_messenger" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f8e4e8975dcd2dbf94c7f84409096f0a0e5af287eabc9a212704e9e325ec84" +checksum = "a94c75e3178528c45bca0b7d515bdc6a3733b5f67d5b37b8a085cf6d7f81e3f2" dependencies = [ "async-mutex", "async-trait", @@ -4091,6 +4165,21 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "polling" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.3.9", + "pin-project-lite", + "rustix 0.38.32", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "polyval" version = "0.5.3" @@ -4109,14 +4198,20 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" dependencies = [ - "portable-atomic 1.4.3", + "portable-atomic 1.6.0", ] [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -4143,6 +4238,15 @@ dependencies = [ "toml_edit 0.19.15", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4169,9 +4273,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -4184,7 +4288,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "version_check", "yansi", ] @@ -4282,7 +4386,7 @@ checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -4296,7 +4400,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.10", "thiserror", "tokio", "tracing", @@ -4312,7 +4416,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.10", "rustls-native-certs", "slab", "thiserror", @@ -4328,16 +4432,16 @@ checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ "bytes", "libc", - "socket2 0.5.4", + "socket2 0.5.6", "tracing", "windows-sys 0.48.0", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4445,7 +4549,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", ] [[package]] @@ -4468,9 +4572,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -4478,9 +4582,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -4494,7 +4598,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.29", + "time 0.3.35", "yasna", ] @@ -4516,7 +4620,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes", - "combine 4.6.6", + "combine 4.6.7", "futures", "futures-util", "itoa", @@ -4542,34 +4646,34 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.14", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -4583,13 +4687,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.3", ] [[package]] @@ -4600,27 +4704,27 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "rend" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -4639,11 +4743,12 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.10", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -4654,7 +4759,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", "winreg", ] @@ -4675,26 +4780,28 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.2" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911b295d2d302948838c8ac142da1ee09fa7863163b44e6715bc9357905878b8" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "getrandom 0.2.10", + "cfg-if", + "getrandom 0.2.14", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -4706,9 +4813,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -4751,12 +4858,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.32.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" dependencies = [ "arrayvec", - "borsh 0.10.3", + "borsh 1.4.0", "bytes", "num-traits", "rand 0.8.5", @@ -4779,9 +4886,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-serialize" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" [[package]] name = "rustc_version" @@ -4817,15 +4924,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.10", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] @@ -4842,12 +4949,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -4866,49 +4973,49 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" dependencies = [ "dyn-clone", "schemars_derive", @@ -4918,9 +5025,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ "proc-macro2", "quote", @@ -4951,17 +5058,17 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -4986,7 +5093,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", - "time 0.3.29", + "time 0.3.35", "tracing", "url", "uuid", @@ -5047,7 +5154,7 @@ dependencies = [ "rust_decimal", "sea-query-derive 0.2.0", "serde_json", - "time 0.3.29", + "time 0.3.35", "uuid", ] @@ -5071,7 +5178,7 @@ dependencies = [ "sea-query 0.27.2", "serde_json", "sqlx", - "time 0.3.29", + "time 0.3.35", "uuid", ] @@ -5154,9 +5261,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -5167,9 +5274,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -5177,37 +5284,37 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -5223,11 +5330,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5235,9 +5342,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -5266,20 +5373,20 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.2", + "indexmap 2.2.6", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.6.1", - "time 0.3.29", + "serde_with_macros 3.7.0", + "time 0.3.35", ] [[package]] @@ -5291,28 +5398,28 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5341,7 +5448,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -5458,9 +5565,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "similar" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" [[package]] name = "sized-chunks" @@ -5483,15 +5590,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -5499,12 +5606,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5525,12 +5632,12 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" +checksum = "d145d4e1e33bfecd209059a0c4c75d623dbcaeb565b4c197f1815257be45726a" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "bv", @@ -5550,9 +5657,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" +checksum = "1deaf83f98be3ba9ecee057efa5cdfa6112267e5b1ff53c4ef4b727f66090b9a" dependencies = [ "chrono", "clap 2.34.0", @@ -5567,16 +5674,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" +checksum = "2a8912026a203ff0d90973e7363f141c6ce569484e06ee0a6f72992144263136" dependencies = [ "async-trait", "bincode", "dashmap 4.0.2", "futures", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "quinn", @@ -5600,9 +5707,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" +checksum = "4000f4717f86c5f9e1105378e3a6521db770d0ad68417f59960ca4b51103fcd0" dependencies = [ "bincode", "chrono", @@ -5614,15 +5721,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" +checksum = "f8b1a4d67c01e5c4464ed9bffb656dec2c88e9bfd7d45bf754f650150e5d81db" dependencies = [ "async-trait", "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "log", "rand 0.8.5", "rayon", @@ -5636,9 +5743,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" +checksum = "1790013c7969353000c22907fc21610adb3389a7c9a27a386ebe7fb32b2ad307" dependencies = [ "ahash 0.8.5", "blake3", @@ -5666,21 +5773,21 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" +checksum = "a3ed2b49a3dd03ddd5107d6e629e8e5895724227a057b3511bf0c107c6d48308" dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "solana-logger" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" +checksum = "bfc0d5b4f046d07e845b69178989a6b3bf168a82eeee006adb77391b339bce64" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5689,9 +5796,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" +checksum = "857178177c6b378bcfc35df6867a6eef211059f5e4ab01ee87355d6b7493b556" dependencies = [ "log", "solana-sdk", @@ -5699,9 +5806,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" +checksum = "1c68f5cbfbafd002b4d94728748f632a3bd27772ca5c7139710d65940c95477c" dependencies = [ "crossbeam-channel", "gethostname", @@ -5714,9 +5821,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" +checksum = "8ce93c50199f077df9d8fd4a82bbdfe654422e1d21e5eecb7f878c4e0078f92f" dependencies = [ "bincode", "clap 3.2.25", @@ -5726,7 +5833,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_derive", - "socket2 0.5.4", + "socket2 0.5.6", "solana-logger", "solana-sdk", "solana-version", @@ -5736,9 +5843,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" +checksum = "2a233bc2032f4b3f462fe13325cd1a336f355e1d9e793faada488db409cec0d6" dependencies = [ "ahash 0.8.5", "bincode", @@ -5765,17 +5872,17 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" +checksum = "4b2ae4ec9dd6fc76202c94d23942da3cf624325a178e1b0125e70db90b8d7f15" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.5.0", "blake3", "borsh 0.10.3", "borsh 0.9.3", @@ -5786,7 +5893,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.2.10", + "getrandom 0.2.14", "itertools 0.10.5", "js-sys", "lazy_static", @@ -5794,7 +5901,7 @@ dependencies = [ "libsecp256k1", "light-poseidon", "log", - "memoffset 0.9.0", + "memoffset 0.9.1", "num-bigint 0.4.4", "num-derive 0.3.3", "num-traits", @@ -5819,11 +5926,11 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" +checksum = "b50a6da7b501117f68ef51fc113d771b52af646dc42c43af23a85e32461d59c9" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bincode", "eager", "enum-iterator", @@ -5847,9 +5954,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" +checksum = "bf920143eb7d5bd5f2ea8fc6ab8a1f521512dfe50af7833af40d7cbae83d955d" dependencies = [ "crossbeam-channel", "futures-util", @@ -5872,9 +5979,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" +checksum = "627491c0afd615efb2538c8a49948663ac01aadf99a3cfebb0a63e2b9431ed79" dependencies = [ "async-mutex", "async-trait", @@ -5885,7 +5992,7 @@ dependencies = [ "quinn", "quinn-proto", "rcgen", - "rustls 0.21.7", + "rustls 0.21.10", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -5899,9 +6006,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" +checksum = "4d5c306f32e26031c043c218a0ba3cde08cbb0e08511ab8a4128445d92a535e0" dependencies = [ "lazy_static", "num_cpus", @@ -5909,9 +6016,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" +checksum = "f1c9dbb8cca1b917a01979a1ca98b682f6123df56f11a5227a37c36494ad7740" dependencies = [ "console", "dialoguer", @@ -5928,12 +6035,12 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" +checksum = "2edc8c0f26561e770f28edb9d456221a5554ee90eeb9052ed38942dbb31c035b" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "indicatif", @@ -5954,11 +6061,11 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" +checksum = "5ff63ab63805a6302ec21284a1138daf5478020f79641d77e53bb84f16c1f5a6" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", "reqwest", @@ -5976,9 +6083,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" +checksum = "897db0c1085b8a99ca35871132b6dc6dca3eff68d9210fcd168ccc2e3843dc26" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -5989,14 +6096,14 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" +checksum = "368430d6c9f033e86f8f590d19232d10986d1188c3ad3a6836628d2acc09c21a" dependencies = [ "assert_matches", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.5.0", "borsh 0.10.3", "bs58 0.4.0", "bytemuck", @@ -6043,15 +6150,15 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" +checksum = "f554d2a144bb0138cfdeced9961cc8a09aaa09f0c3c9a63bd10da41c4a06d420" dependencies = [ "bs58 0.4.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6062,16 +6169,16 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" +checksum = "e28e8941bc0b137f851626ff4d4ce9ebcf9be10619b48d2ed83cd1953af21821" dependencies = [ "async-channel 1.9.0", "bytes", "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.0.2", + "indexmap 2.2.6", "itertools 0.10.5", "libc", "log", @@ -6083,7 +6190,7 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rcgen", - "rustls 0.21.7", + "rustls 0.21.10", "solana-metrics", "solana-perf", "solana-sdk", @@ -6094,9 +6201,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" +checksum = "760b94e2875b54350f786c85faf50fa1a9a37ae46e9897215d19b373fc2f58cd" dependencies = [ "bincode", "log", @@ -6109,14 +6216,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" +checksum = "7cfdc7ff6cdc1512661de1f9f40723f88dc1e94c8b8938bd537f3713239173e2" dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "rayon", @@ -6133,12 +6240,12 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" +checksum = "ba7131d11c8d5a068bfc26a9dc8c9ee0d77eaf60856dd0c8be880542fc5fbbd6" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "borsh 0.10.3", "bs58 0.4.0", @@ -6158,9 +6265,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" +checksum = "54647340d7fa1708c217cbc2411259c5b3784c2df55c1eb4353aca296635ed87" dependencies = [ "async-trait", "solana-connection-cache", @@ -6173,9 +6280,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" +checksum = "1de7a6bad7dff1c595be2eec0c3800a482c6068f3c87c6df87ed091b4e6e642e" dependencies = [ "log", "rustc_version", @@ -6189,9 +6296,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" +checksum = "3c828d118d1f135baacfbf51412c4f1ac18af19fdbee43b002d2908e69cdf50b" dependencies = [ "bincode", "log", @@ -6211,12 +6318,12 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" +checksum = "112944743b08f7e1101368ff6d84745e7b4abb075fabaccc02e01bd3ce4b6d6c" dependencies = [ "aes-gcm-siv", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bytemuck", "byteorder", @@ -6299,7 +6406,7 @@ checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f" dependencies = [ "assert_matches", "borsh 0.10.3", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "solana-program", "spl-token", @@ -6320,9 +6427,9 @@ dependencies = [ [[package]] name = "spl-discriminator" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f" +checksum = "daa600f2fe56f32e923261719bae640d873edadbc5237681a39b8e37bfd4d263" dependencies = [ "bytemuck", "solana-program", @@ -6331,25 +6438,25 @@ dependencies = [ [[package]] name = "spl-discriminator-derive" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" +checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" dependencies = [ "quote", "spl-discriminator-syn", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "spl-discriminator-syn" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" +checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.38", + "syn 2.0.58", "thiserror", ] @@ -6373,11 +6480,11 @@ dependencies = [ [[package]] name = "spl-pod" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" +checksum = "85a5db7e4efb1107b0b8e52a13f035437cdcb36ef99c58f6d467f089d9b2915a" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "borsh 0.10.3", "bytemuck", "serde", @@ -6388,11 +6495,11 @@ dependencies = [ [[package]] name = "spl-program-error" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" +checksum = "7e0657b6490196971d9e729520ba934911ff41fbb2cb9004463dbe23cf8b4b4f" dependencies = [ - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "solana-program", "spl-program-error-derive", @@ -6401,14 +6508,14 @@ dependencies = [ [[package]] name = "spl-program-error-derive" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" +checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6448,7 +6555,7 @@ checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059" dependencies = [ "arrayref", "bytemuck", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "num_enum 0.7.2", "solana-program", @@ -6509,9 +6616,9 @@ dependencies = [ [[package]] name = "spl-type-length-value" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac" +checksum = "8f9ebd75d29c5f48de5f6a9c114e08531030b75b8ac2c557600ac7da0b73b1e8" dependencies = [ "bytemuck", "solana-program", @@ -6522,11 +6629,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.11.0", + "itertools 0.12.1", "nom", "unicode_categories", ] @@ -6547,7 +6654,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "atoi", "base64 0.13.1", "bitflags 1.3.2", @@ -6591,7 +6698,7 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", - "time 0.3.29", + "time 0.3.35", "tokio-stream", "url", "uuid", @@ -6645,7 +6752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bcb902b974bc20b50c3ad3148022a366a46c9a676b587684ff46c237a3329e" dependencies = [ "async-channel 1.9.0", - "async-io", + "async-io 1.13.0", "atomic 0.5.3", "crossbeam-channel", "futures", @@ -6680,6 +6787,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.4.1" @@ -6699,15 +6812,33 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -6749,22 +6880,21 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.18", - "windows-sys 0.48.0", + "fastrand 2.0.2", + "rustix 0.38.32", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -6780,35 +6910,35 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -6827,12 +6957,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "ef89ece63debf11bc32d1ed8d078ac870cbeb44da02afb02a9ff135ae7ca0582" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -6846,10 +6978,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -6889,9 +7022,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -6901,7 +7034,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.6", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -6909,13 +7042,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6945,15 +7078,15 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.10", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -6968,18 +7101,18 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.7", + "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", "tungstenite", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", ] [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -7001,14 +7134,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit 0.22.9", ] [[package]] @@ -7026,22 +7159,33 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "toml_datetime", - "winnow", + "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +dependencies = [ + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.5", ] [[package]] @@ -7087,11 +7231,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -7100,20 +7243,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -7168,9 +7311,9 @@ dependencies = [ "anchor-client", "anyhow", "bs58 0.4.0", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "flatbuffers", "futures", "hex", @@ -7192,9 +7335,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" dependencies = [ "serde", "stable_deref_trait", @@ -7202,9 +7345,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -7219,7 +7362,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.7", + "rustls 0.21.10", "sha1", "thiserror", "url", @@ -7232,8 +7375,8 @@ name = "txn_forwarder" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.4", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -7260,9 +7403,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uncased" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" dependencies = [ "version_check", ] @@ -7278,9 +7421,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -7290,18 +7433,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" @@ -7342,9 +7485,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -7370,9 +7513,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -7393,11 +7536,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", "serde", ] @@ -7409,9 +7552,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.2" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "74797339c3b98616c009c7c3eb53a0ce41e85c8ec66bd3db96ed132d20cfdee8" [[package]] name = "vcpkg" @@ -7476,11 +7619,17 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7488,24 +7637,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -7515,9 +7664,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7525,28 +7674,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -7558,7 +7707,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.2", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7582,9 +7731,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "wg" @@ -7599,11 +7748,12 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.4.1", + "wasite", "web-sys", ] @@ -7639,21 +7789,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-targets 0.52.4", ] [[package]] @@ -7666,18 +7807,12 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.52.4", ] [[package]] @@ -7696,10 +7831,19 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-targets" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] [[package]] name = "windows_aarch64_gnullvm" @@ -7708,10 +7852,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -7720,10 +7864,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -7732,10 +7876,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -7744,10 +7888,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -7756,10 +7900,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -7768,10 +7912,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -7779,11 +7923,26 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" -version = "0.5.16" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" dependencies = [ "memchr", ] @@ -7822,29 +7981,20 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.29", + "time 0.3.35", ] [[package]] name = "xxhash-rust" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" - -[[package]] -name = "yaml-rust" -version = "0.4.5" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] +checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" [[package]] name = "yansi" -version = "1.0.0-rc.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yasna" @@ -7852,7 +8002,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.29", + "time 0.3.35", ] [[package]] @@ -7872,7 +8022,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -7892,7 +8042,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -7916,11 +8066,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/migration/src/m20230131_140613_change_token_account_indexes.rs b/migration/src/m20230131_140613_change_token_account_indexes.rs index 3d28256cf..49c84eb33 100644 --- a/migration/src/m20230131_140613_change_token_account_indexes.rs +++ b/migration/src/m20230131_140613_change_token_account_indexes.rs @@ -89,3 +89,14 @@ impl MigrationTrait for Migration { Ok(()) } } +<<<<<<< HEAD +======= + +/// Learn more at https://docs.rs/sea-query#iden +#[allow(dead_code)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Iden)] +enum Index { + BRIN, +} +>>>>>>> bb2eb9c (include migration to workspace) From 7a870c2e355c54cf816edae73057cebb6cc03504 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 20 Nov 2023 13:16:22 -0500 Subject: [PATCH 05/72] include das_api to workspace --- .github/workflows/build-api.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index 8082f52a9..f8e7fd198 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -46,10 +46,6 @@ jobs: - name: build digital asset rpc infra run: cargo build --verbose --release - - name: build das_api - working-directory: das_api - run: cargo build --verbose --release - - name: rename binaries for ubuntu22 release if: matrix.os == 'ubuntu-22.04' run: | From 7e3f9ef84327471666e48ae931ae621a5cb10ea9 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 20 Nov 2023 13:50:36 -0500 Subject: [PATCH 06/72] add global clippy lints --- Cargo.toml | 2 +- das_api/Cargo.toml | 28 ++++++++++++----- migration/Cargo.toml | 5 +++- .../src/program_transformers/bubblegum/db.rs | 2 +- tools/bgtask_creator/Cargo.toml | 22 ++++++++++++-- tools/load_generation/Cargo.toml | 4 ++- tools/tree-status/Cargo.toml | 30 ++++++++++++++++--- tools/txn_forwarder/Cargo.toml | 8 ++++- 8 files changed, 82 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 769cf02a4..6449d1d48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "core", "das_api", @@ -16,7 +17,6 @@ members = [ "tools/tree-status", "tools/txn_forwarder", ] -resolver = "2" [workspace.package] edition = "2021" diff --git a/das_api/Cargo.toml b/das_api/Cargo.toml index 566ac1cd8..d41028193 100644 --- a/das_api/Cargo.toml +++ b/das_api/Cargo.toml @@ -12,25 +12,39 @@ blockbuster = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } -env_logger = { workspace = true } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } +env_logger = { workspace = true } figment = { workspace = true, features = ["env"] } hyper = { workspace = true } -jsonrpsee = { workspace = true, features = ["server", "macros"]} -jsonrpsee-core = { workspace = true, features =["server"]} +jsonrpsee = { workspace = true, features = ["server", "macros"] } +jsonrpsee-core = { workspace = true, features = ["server"] } log = { workspace = true } metrics = { workspace = true } mpl-bubblegum = { workspace = true } -mpl-token-metadata = { workspace = true, features = ["serde"] } +mpl-token-metadata = { workspace = true, features = ["serde"] } open-rpc-derive = { workspace = true } open-rpc-schema = { workspace = true } schemars = { workspace = true } schemars_derive = { workspace = true } -sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres"] } +sea-orm = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "sqlx-postgres", +] } serde = { workspace = true } serde_json = { workspace = true } solana-sdk = { workspace = true } -sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } thiserror = { workspace = true } tokio = { workspace = true, features = ["signal"] } tower = { workspace = true } diff --git a/migration/Cargo.toml b/migration/Cargo.toml index c0202ffdd..6ce0612be 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -9,7 +9,10 @@ publish = { workspace = true } async-std = { workspace = true, features = ["attributes", "tokio1"] } enum-iterator = { workspace = true } enum-iterator-derive = { workspace = true } -sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres"] } +sea-orm-migration = { workspace = true, features = [ + "runtime-tokio-rustls", + "sqlx-postgres", +] } [lints] workspace = true diff --git a/nft_ingester/src/program_transformers/bubblegum/db.rs b/nft_ingester/src/program_transformers/bubblegum/db.rs index c562be2be..b7141e231 100644 --- a/nft_ingester/src/program_transformers/bubblegum/db.rs +++ b/nft_ingester/src/program_transformers/bubblegum/db.rs @@ -23,7 +23,7 @@ where Ok(change_log_event.seq) } -fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { +const fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { index - 2i64.pow(tree_height) } diff --git a/tools/bgtask_creator/Cargo.toml b/tools/bgtask_creator/Cargo.toml index 7011185e5..be662e34e 100644 --- a/tools/bgtask_creator/Cargo.toml +++ b/tools/bgtask_creator/Cargo.toml @@ -8,16 +8,32 @@ publish = { workspace = true } [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "cargo"] } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } futures = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } nft_ingester = { workspace = true } prometheus = { workspace = true } -sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] } +sea-orm = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "sqlx-postgres", + "with-chrono", + "mock", +] } sea-query = { workspace = true, features = ["postgres-array"] } solana-sdk = { workspace = true } -sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } txn_forwarder = { workspace = true } diff --git a/tools/load_generation/Cargo.toml b/tools/load_generation/Cargo.toml index bd7b0de3b..0c7a97d19 100644 --- a/tools/load_generation/Cargo.toml +++ b/tools/load_generation/Cargo.toml @@ -12,7 +12,9 @@ rand = { workspace = true } solana-client = { workspace = true } solana-program = { workspace = true } solana-sdk = { workspace = true } -spl-associated-token-account = { workspace = true, features = ["no-entrypoint"] } +spl-associated-token-account = { workspace = true, features = [ + "no-entrypoint", +] } spl-token = { workspace = true, features = ["no-entrypoint"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/tools/tree-status/Cargo.toml b/tools/tree-status/Cargo.toml index da8ccc058..98e560f23 100644 --- a/tools/tree-status/Cargo.toml +++ b/tools/tree-status/Cargo.toml @@ -12,7 +12,10 @@ anchor-client = { workspace = true } anyhow = { workspace = true } bs58 = { workspace = true } clap = { workspace = true, features = ["derive"] } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } env_logger = { workspace = true } flatbuffers = { workspace = true } futures = { workspace = true } @@ -20,16 +23,35 @@ hex = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } prometheus = { workspace = true } -sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] } +sea-orm = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "sqlx-postgres", + "with-chrono", + "mock", +] } serde_json = { workspace = true } solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true, features = ["no-entrypoint"] } spl-noop = { workspace = true, features = ["no-entrypoint"] } -sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread", "sync", "time"] } +tokio = { workspace = true, features = [ + "fs", + "macros", + "rt-multi-thread", + "sync", + "time", +] } txn_forwarder = { workspace = true } [lints] diff --git a/tools/txn_forwarder/Cargo.toml b/tools/txn_forwarder/Cargo.toml index c9f5ca999..9a8388325 100644 --- a/tools/txn_forwarder/Cargo.toml +++ b/tools/txn_forwarder/Cargo.toml @@ -23,7 +23,13 @@ solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time", "fs"] } +tokio = { workspace = true, features = [ + "macros", + "rt-multi-thread", + "sync", + "time", + "fs", +] } tokio-stream = { workspace = true, features = ["io-util"] } [lints] From aa916a4ecd10d0f719edf25b99e5a68bb7a6f18e Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 20 Nov 2023 15:27:59 -0500 Subject: [PATCH 07/72] use workspace --- Cargo.toml | 2 +- digital_asset_types/Cargo.toml | 14 +++++++++++--- ...30131_140613_change_token_account_indexes.rs | 11 ----------- tools/tree-status/Cargo.toml | 17 +++++++++-------- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6449d1d48..769cf02a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,4 @@ [workspace] -resolver = "2" members = [ "core", "das_api", @@ -17,6 +16,7 @@ members = [ "tools/tree-status", "tools/txn_forwarder", ] +resolver = "2" [workspace.package] edition = "2021" diff --git a/digital_asset_types/Cargo.toml b/digital_asset_types/Cargo.toml index 449f6e78d..74fb32353 100644 --- a/digital_asset_types/Cargo.toml +++ b/digital_asset_types/Cargo.toml @@ -8,7 +8,7 @@ publish = { workspace = true } [dependencies] async-trait = { workspace = true } blockbuster = { workspace = true } -borsh = { workspace = true, optional = true } +borsh = { workspace = true, optional = true } borsh-derive = { workspace = true, optional = true } bs58 = { workspace = true } futures = { workspace = true } @@ -20,10 +20,18 @@ num-derive = { workspace = true } num-traits = { workspace = true } schemars = { workspace = true } schemars_derive = { workspace = true } -sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"], optional = true } +sea-orm = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "sqlx-postgres", + "with-chrono", + "mock", +], optional = true } sea-query = { workspace = true, features = ["postgres-array"] } serde = { workspace = true, optional = true } -serde_json = { workspace = true, features = ["preserve_order"], optional = true } +serde_json = { workspace = true, features = [ + "preserve_order", +], optional = true } solana-sdk = { workspace = true } spl-concurrent-merkle-tree = { workspace = true } thiserror = { workspace = true } diff --git a/migration/src/m20230131_140613_change_token_account_indexes.rs b/migration/src/m20230131_140613_change_token_account_indexes.rs index 49c84eb33..3d28256cf 100644 --- a/migration/src/m20230131_140613_change_token_account_indexes.rs +++ b/migration/src/m20230131_140613_change_token_account_indexes.rs @@ -89,14 +89,3 @@ impl MigrationTrait for Migration { Ok(()) } } -<<<<<<< HEAD -======= - -/// Learn more at https://docs.rs/sea-query#iden -#[allow(dead_code)] -#[allow(clippy::upper_case_acronyms)] -#[derive(Iden)] -enum Index { - BRIN, -} ->>>>>>> bb2eb9c (include migration to workspace) diff --git a/tools/tree-status/Cargo.toml b/tools/tree-status/Cargo.toml index 98e560f23..3ff2e0c37 100644 --- a/tools/tree-status/Cargo.toml +++ b/tools/tree-status/Cargo.toml @@ -8,6 +8,7 @@ repository = { workspace = true } publish = { workspace = true } [dependencies] + anchor-client = { workspace = true } anyhow = { workspace = true } bs58 = { workspace = true } @@ -36,14 +37,6 @@ solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true, features = ["no-entrypoint"] } spl-noop = { workspace = true, features = ["no-entrypoint"] } -sqlx = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "postgres", - "uuid", - "offline", - "json", -] } thiserror = { workspace = true } tokio = { workspace = true, features = [ "fs", @@ -52,6 +45,14 @@ tokio = { workspace = true, features = [ "sync", "time", ] } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } txn_forwarder = { workspace = true } [lints] From 033873d216c28dda0d26121ab903ee656df5f64c Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 22 Nov 2023 12:10:23 -0500 Subject: [PATCH 08/72] add crate program_transformers --- .../src/bubblegum/decompress.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 program_transformers/src/bubblegum/decompress.rs diff --git a/program_transformers/src/bubblegum/decompress.rs b/program_transformers/src/bubblegum/decompress.rs new file mode 100644 index 000000000..9db67e8b8 --- /dev/null +++ b/program_transformers/src/bubblegum/decompress.rs @@ -0,0 +1,34 @@ +use { + crate::{ + bubblegum::db::{ + upsert_asset_with_compression_info, upsert_asset_with_leaf_info_for_decompression, + }, + error::ProgramTransformerResult, + }, + blockbuster::{instruction::InstructionBundle, programs::bubblegum::BubblegumInstruction}, + sea_orm::{ConnectionTrait, TransactionTrait}, +}; + +pub async fn decompress<'c, T>( + _parsing_result: &BubblegumInstruction, + bundle: &InstructionBundle<'c>, + txn: &'c T, +) -> ProgramTransformerResult<()> +where + T: ConnectionTrait + TransactionTrait, +{ + let id_bytes = bundle.keys.get(3).unwrap().0.as_slice(); + + // Partial update of asset table with just leaf. + upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.to_vec()).await?; + upsert_asset_with_compression_info( + txn, + id_bytes.to_vec(), + false, + false, + 1, + Some(id_bytes.to_vec()), + true, + ) + .await +} From 1008f862e41841f0193d0060abcb9cfdaadc54a3 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 22 Nov 2023 22:04:05 -0500 Subject: [PATCH 09/72] nft_ingester: use program_transformers crate --- nft_ingester/Cargo.toml | 1 + nft_ingester/src/error/mod.rs | 50 +- .../src/program_transformers/bubblegum/db.rs | 442 ------------------ .../program_transformers/bubblegum/mint_v1.rs | 364 --------------- nft_ingester/src/program_transformers/mod.rs | 189 -------- .../src/program_transformers/token/mod.rs | 141 ------ .../token_metadata/v1_asset.rs | 393 ---------------- 7 files changed, 7 insertions(+), 1573 deletions(-) delete mode 100644 nft_ingester/src/program_transformers/bubblegum/db.rs delete mode 100644 nft_ingester/src/program_transformers/bubblegum/mint_v1.rs delete mode 100644 nft_ingester/src/program_transformers/mod.rs delete mode 100644 nft_ingester/src/program_transformers/token/mod.rs delete mode 100644 nft_ingester/src/program_transformers/token_metadata/v1_asset.rs diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index fd17a2bed..db41c0a1f 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -6,6 +6,7 @@ repository = { workspace = true } publish = { workspace = true } [dependencies] + async-trait = { workspace = true } blockbuster = { workspace = true } borsh = { workspace = true } diff --git a/nft_ingester/src/error/mod.rs b/nft_ingester/src/error/mod.rs index 37ed5f24b..ddeae0ba6 100644 --- a/nft_ingester/src/error/mod.rs +++ b/nft_ingester/src/error/mod.rs @@ -1,27 +1,15 @@ -use crate::tasks::TaskData; -use blockbuster::error::BlockbusterError; -use plerkle_messenger::MessengerError; -use plerkle_serialization::error::PlerkleSerializationError; -use sea_orm::{DbErr, TransactionError}; -use thiserror::Error; -use tokio::sync::mpsc::error::SendError; +use { + crate::tasks::TaskData, plerkle_messenger::MessengerError, + plerkle_serialization::error::PlerkleSerializationError, sea_orm::DbErr, + tokio::sync::mpsc::error::SendError, +}; -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, thiserror::Error)] pub enum IngesterError { - #[error("ChangeLog Event Malformed")] - ChangeLogEventMalformed, - #[error("Compressed Asset Event Malformed")] - CompressedAssetEventMalformed, #[error("Network Error: {0}")] BatchInitNetworkingError(String), - #[error("Error writing batch files")] - BatchInitIOError, - #[error("Storage listener error: ({msg})")] - StorageListenerError { msg: String }, #[error("Storage Write Error: {0}")] StorageWriteError(String), - #[error("NotImplemented")] - NotImplemented, #[error("Deserialization Error: {0}")] DeserializationError(String), #[error("Task Manager Error: {0}")] @@ -36,12 +24,6 @@ pub enum IngesterError { SerializatonError(String), #[error("Messenger error; {0}")] MessengerError(String), - #[error("Blockbuster Parsing error: {0}")] - ParsingError(String), - #[error("Database Error: {0}")] - DatabaseError(String), - #[error("Unknown Task Type: {0}")] - UnknownTaskType(String), #[error("BG Task Manager Not Started")] TaskManagerNotStarted, #[error("Unrecoverable task error: {0}")] @@ -50,8 +32,6 @@ pub enum IngesterError { CacheStorageWriteError(String), #[error("HttpError {status_code}")] HttpError { status_code: String }, - #[error("AssetIndex Error {0}")] - AssetIndexError(String), } impl From for IngesterError { @@ -72,30 +52,12 @@ impl From for IngesterError { } } -impl From for IngesterError { - fn from(err: BlockbusterError) -> Self { - IngesterError::ParsingError(err.to_string()) - } -} - -impl From for IngesterError { - fn from(_err: std::io::Error) -> Self { - IngesterError::BatchInitIOError - } -} - impl From for IngesterError { fn from(e: DbErr) -> Self { IngesterError::StorageWriteError(e.to_string()) } } -impl From> for IngesterError { - fn from(e: TransactionError) -> Self { - IngesterError::StorageWriteError(e.to_string()) - } -} - impl From> for IngesterError { fn from(err: SendError) -> Self { IngesterError::TaskManagerError(format!("Could not create task: {:?}", err.to_string())) diff --git a/nft_ingester/src/program_transformers/bubblegum/db.rs b/nft_ingester/src/program_transformers/bubblegum/db.rs deleted file mode 100644 index b7141e231..000000000 --- a/nft_ingester/src/program_transformers/bubblegum/db.rs +++ /dev/null @@ -1,442 +0,0 @@ -use crate::error::IngesterError; -use digital_asset_types::dao::{ - asset, asset_creators, asset_grouping, backfill_items, cl_audits, cl_items, -}; -use log::{debug, info}; -use mpl_bubblegum::types::Collection; -use sea_orm::{ - query::*, sea_query::OnConflict, ActiveValue::Set, ColumnTrait, DbBackend, EntityTrait, -}; -use spl_account_compression::events::ChangeLogEventV1; - -pub async fn save_changelog_event<'c, T>( - change_log_event: &ChangeLogEventV1, - slot: u64, - txn_id: &str, - txn: &T, - cl_audits: bool, -) -> Result -where - T: ConnectionTrait + TransactionTrait, -{ - insert_change_log(change_log_event, slot, txn_id, txn, cl_audits).await?; - Ok(change_log_event.seq) -} - -const fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { - index - 2i64.pow(tree_height) -} - -pub async fn insert_change_log<'c, T>( - change_log_event: &ChangeLogEventV1, - slot: u64, - txn_id: &str, - txn: &T, - cl_audits: bool, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let mut i: i64 = 0; - let depth = change_log_event.path.len() - 1; - let tree_id = change_log_event.id.as_ref(); - for p in change_log_event.path.iter() { - let node_idx = p.index as i64; - debug!( - "seq {}, index {} level {}, node {:?}, txn: {:?}", - change_log_event.seq, - p.index, - i, - bs58::encode(p.node).into_string(), - txn_id, - ); - let leaf_idx = if i == 0 { - Some(node_idx_to_leaf_idx(node_idx, depth as u32)) - } else { - None - }; - - let item = cl_items::ActiveModel { - tree: Set(tree_id.to_vec()), - level: Set(i), - node_idx: Set(node_idx), - hash: Set(p.node.as_ref().to_vec()), - seq: Set(change_log_event.seq as i64), - leaf_idx: Set(leaf_idx), - ..Default::default() - }; - - let audit_item: Option = if cl_audits { - let mut ai: cl_audits::ActiveModel = item.clone().into(); - ai.tx = Set(txn_id.to_string()); - Some(ai) - } else { - None - }; - - i += 1; - let mut query = cl_items::Entity::insert(item) - .on_conflict( - OnConflict::columns([cl_items::Column::Tree, cl_items::Column::NodeIdx]) - .update_columns([ - cl_items::Column::Hash, - cl_items::Column::Seq, - cl_items::Column::LeafIdx, - cl_items::Column::Level, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!("{} WHERE excluded.seq > cl_items.seq", query.sql); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - // Insert the audit item after the insert into cl_items have been completed - if let Some(audit_item) = audit_item { - cl_audits::Entity::insert(audit_item).exec(txn).await?; - } - } - - // If and only if the entire path of nodes was inserted into the `cl_items` table, then insert - // a single row into the `backfill_items` table. This way if an incomplete path was inserted - // into `cl_items` due to an error, a gap will be created for the tree and the backfiller will - // fix it. - if i - 1 == depth as i64 { - // See if the tree already exists in the `backfill_items` table. - let rows = backfill_items::Entity::find() - .filter(backfill_items::Column::Tree.eq(tree_id)) - .limit(1) - .all(txn) - .await?; - - // If the tree does not exist in `backfill_items` and the sequence number is greater than 1, - // then we know we will need to backfill the tree from sequence number 1 up to the current - // sequence number. So in this case we set at flag to force checking the tree. - let force_chk = rows.is_empty() && change_log_event.seq > 1; - - info!("Adding to backfill_items table at level {}", i - 1); - let item = backfill_items::ActiveModel { - tree: Set(tree_id.to_vec()), - seq: Set(change_log_event.seq as i64), - slot: Set(slot as i64), - force_chk: Set(force_chk), - backfilled: Set(false), - failed: Set(false), - ..Default::default() - }; - - backfill_items::Entity::insert(item).exec(txn).await?; - } - - Ok(()) - //TODO -> set maximum size of path and break into multiple statements -} - -#[allow(clippy::too_many_arguments)] -pub async fn upsert_asset_with_leaf_info( - txn: &T, - id: Vec, - nonce: i64, - tree_id: Vec, - leaf: Vec, - data_hash: [u8; 32], - creator_hash: [u8; 32], - seq: i64, - was_decompressed: bool, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let data_hash = bs58::encode(data_hash).into_string().trim().to_string(); - let creator_hash = bs58::encode(creator_hash).into_string().trim().to_string(); - let model = asset::ActiveModel { - id: Set(id), - nonce: Set(Some(nonce)), - tree_id: Set(Some(tree_id)), - leaf: Set(Some(leaf)), - data_hash: Set(Some(data_hash)), - creator_hash: Set(Some(creator_hash)), - leaf_seq: Set(Some(seq)), - ..Default::default() - }; - - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::column(asset::Column::Id) - .update_columns([ - asset::Column::Nonce, - asset::Column::TreeId, - asset::Column::Leaf, - asset::Column::LeafSeq, - asset::Column::DataHash, - asset::Column::CreatorHash, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - - // If we are indexing decompression we will update the leaf regardless of if we have previously - // indexed decompression and regardless of seq. - if !was_decompressed { - query.sql = format!( - "{} WHERE (NOT asset.was_decompressed) AND (excluded.leaf_seq >= asset.leaf_seq OR asset.leaf_seq IS NULL)", - query.sql - ); - } - - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} - -pub async fn upsert_asset_with_leaf_info_for_decompression( - txn: &T, - id: Vec, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let model = asset::ActiveModel { - id: Set(id), - leaf: Set(None), - nonce: Set(Some(0)), - leaf_seq: Set(None), - data_hash: Set(None), - creator_hash: Set(None), - tree_id: Set(None), - seq: Set(Some(0)), - ..Default::default() - }; - let query = asset::Entity::insert(model) - .on_conflict( - OnConflict::column(asset::Column::Id) - .update_columns([ - asset::Column::Leaf, - asset::Column::LeafSeq, - asset::Column::Nonce, - asset::Column::DataHash, - asset::Column::CreatorHash, - asset::Column::TreeId, - asset::Column::Seq, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} - -pub async fn upsert_asset_with_owner_and_delegate_info( - txn: &T, - id: Vec, - owner: Vec, - delegate: Option>, - seq: i64, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let model = asset::ActiveModel { - id: Set(id), - owner: Set(Some(owner)), - delegate: Set(delegate), - owner_delegate_seq: Set(Some(seq)), // gummyroll seq - ..Default::default() - }; - - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::column(asset::Column::Id) - .update_columns([ - asset::Column::Owner, - asset::Column::Delegate, - asset::Column::OwnerDelegateSeq, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.owner_delegate_seq >= asset.owner_delegate_seq OR asset.owner_delegate_seq IS NULL", - query.sql - ); - - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} - -pub async fn upsert_asset_with_compression_info( - txn: &T, - id: Vec, - compressed: bool, - compressible: bool, - supply: i64, - supply_mint: Option>, - was_decompressed: bool, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let model = asset::ActiveModel { - id: Set(id), - compressed: Set(compressed), - compressible: Set(compressible), - supply: Set(supply), - supply_mint: Set(supply_mint), - was_decompressed: Set(was_decompressed), - ..Default::default() - }; - - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::columns([asset::Column::Id]) - .update_columns([ - asset::Column::Compressed, - asset::Column::Compressible, - asset::Column::Supply, - asset::Column::SupplyMint, - asset::Column::WasDecompressed, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!("{} WHERE NOT asset.was_decompressed", query.sql); - txn.execute(query).await?; - - Ok(()) -} - -pub async fn upsert_asset_with_seq(txn: &T, id: Vec, seq: i64) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let model = asset::ActiveModel { - id: Set(id), - seq: Set(Some(seq)), - ..Default::default() - }; - - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::column(asset::Column::Id) - .update_columns([asset::Column::Seq]) - .to_owned(), - ) - .build(DbBackend::Postgres); - - query.sql = format!( - "{} WHERE (NOT asset.was_decompressed) AND (excluded.seq >= asset.seq OR asset.seq IS NULL)", - query.sql - ); - - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} - -pub async fn upsert_creator_verified( - txn: &T, - asset_id: Vec, - creator: Vec, - verified: bool, - seq: i64, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let model = asset_creators::ActiveModel { - asset_id: Set(asset_id), - creator: Set(creator), - verified: Set(verified), - seq: Set(Some(seq)), - ..Default::default() - }; - - let mut query = asset_creators::Entity::insert(model) - .on_conflict( - OnConflict::columns([ - asset_creators::Column::AssetId, - asset_creators::Column::Creator, - ]) - .update_columns([ - asset_creators::Column::Verified, - asset_creators::Column::Seq, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - - query.sql = format!( - "{} WHERE excluded.seq >= asset_creators.seq OR asset_creators.seq is NULL", - query.sql - ); - - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} - -pub async fn upsert_collection_info( - txn: &T, - asset_id: Vec, - collection: Option, - slot_updated: i64, - seq: i64, -) -> Result<(), IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - let (group_value, verified) = match collection { - Some(c) => (Some(c.key.to_string()), c.verified), - None => (None, false), - }; - - let model = asset_grouping::ActiveModel { - asset_id: Set(asset_id), - group_key: Set("collection".to_string()), - group_value: Set(group_value), - verified: Set(Some(verified)), - slot_updated: Set(Some(slot_updated)), - group_info_seq: Set(Some(seq)), - ..Default::default() - }; - - let mut query = asset_grouping::Entity::insert(model) - .on_conflict( - OnConflict::columns([ - asset_grouping::Column::AssetId, - asset_grouping::Column::GroupKey, - ]) - .update_columns([ - asset_grouping::Column::GroupValue, - asset_grouping::Column::Verified, - asset_grouping::Column::SlotUpdated, - asset_grouping::Column::GroupInfoSeq, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - - query.sql = format!( - "{} WHERE excluded.group_info_seq >= asset_grouping.group_info_seq OR asset_grouping.group_info_seq IS NULL", - query.sql - ); - - txn.execute(query) - .await - .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?; - - Ok(()) -} diff --git a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs deleted file mode 100644 index 685649f79..000000000 --- a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs +++ /dev/null @@ -1,364 +0,0 @@ -use crate::{ - error::IngesterError, - program_transformers::bubblegum::{ - save_changelog_event, upsert_asset_with_compression_info, upsert_asset_with_leaf_info, - upsert_asset_with_owner_and_delegate_info, upsert_asset_with_seq, upsert_collection_info, - }, - tasks::{DownloadMetadata, IntoTaskData, TaskData}, -}; -use blockbuster::{ - instruction::InstructionBundle, - programs::bubblegum::{BubblegumInstruction, LeafSchema, Payload}, - token_metadata::{ - pda::find_master_edition_account, - state::{TokenStandard, UseMethod, Uses}, - }, -}; -use chrono::Utc; -use digital_asset_types::{ - dao::{ - asset, asset_authority, asset_creators, asset_data, asset_v1_account_attachments, - sea_orm_active_enums::{ChainMutability, Mutability, OwnerType, RoyaltyTargetType}, - }, - json::ChainDataV1, -}; -use log::warn; -use num_traits::FromPrimitive; -use sea_orm::{ - entity::*, query::*, sea_query::OnConflict, ConnectionTrait, DbBackend, EntityTrait, JsonValue, -}; -use std::collections::HashSet; - -use digital_asset_types::dao::sea_orm_active_enums::{ - SpecificationAssetClass, SpecificationVersions, V1AccountAttachments, -}; - -// TODO -> consider moving structs into these functions to avoid clone - -pub async fn mint_v1<'c, T>( - parsing_result: &BubblegumInstruction, - bundle: &InstructionBundle<'c>, - txn: &'c T, - cl_audits: bool, -) -> Result, IngesterError> -where - T: ConnectionTrait + TransactionTrait, -{ - if let (Some(le), Some(cl), Some(Payload::MintV1 { args })) = ( - &parsing_result.leaf_update, - &parsing_result.tree_update, - &parsing_result.payload, - ) { - let seq = save_changelog_event(cl, bundle.slot, bundle.txn_id, txn, cl_audits).await?; - let metadata = args; - #[allow(unreachable_patterns)] - return match le.schema { - LeafSchema::V1 { - id, - delegate, - owner, - nonce, - .. - } => { - let (edition_attachment_address, _) = find_master_edition_account(&id); - let id_bytes = id.to_bytes(); - let slot_i = bundle.slot as i64; - let uri = metadata.uri.replace('\0', ""); - let name = metadata.name.clone().into_bytes(); - let symbol = metadata.symbol.clone().into_bytes(); - let mut chain_data = ChainDataV1 { - name: metadata.name.clone(), - symbol: metadata.symbol.clone(), - edition_nonce: metadata.edition_nonce, - primary_sale_happened: metadata.primary_sale_happened, - token_standard: Some(TokenStandard::NonFungible), - uses: metadata.uses.clone().map(|u| Uses { - use_method: UseMethod::from_u8(u.use_method as u8).unwrap(), - remaining: u.remaining, - total: u.total, - }), - }; - chain_data.sanitize(); - let chain_data_json = serde_json::to_value(chain_data) - .map_err(|e| IngesterError::DeserializationError(e.to_string()))?; - let chain_mutability = match metadata.is_mutable { - true => ChainMutability::Mutable, - false => ChainMutability::Immutable, - }; - - let data = asset_data::ActiveModel { - id: Set(id_bytes.to_vec()), - chain_data_mutability: Set(chain_mutability), - chain_data: Set(chain_data_json), - metadata_url: Set(uri.clone()), - metadata: Set(JsonValue::String("processing".to_string())), - metadata_mutability: Set(Mutability::Mutable), - slot_updated: Set(slot_i), - reindex: Set(Some(true)), - raw_name: Set(name.to_vec()), - raw_symbol: Set(symbol.to_vec()), - }; - - let mut query = asset_data::Entity::insert(data) - .on_conflict( - OnConflict::columns([asset_data::Column::Id]) - .update_columns([ - asset_data::Column::ChainDataMutability, - asset_data::Column::ChainData, - asset_data::Column::MetadataUrl, - asset_data::Column::MetadataMutability, - asset_data::Column::SlotUpdated, - asset_data::Column::Reindex, - asset_data::Column::RawName, - asset_data::Column::RawSymbol, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset_data.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - // Insert into `asset` table. - let delegate = if owner == delegate || delegate.to_bytes() == [0; 32] { - None - } else { - Some(delegate.to_bytes().to_vec()) - }; - let tree_id = bundle.keys.get(3).unwrap().0.to_vec(); - - // Set initial mint info. - let asset_model = asset::ActiveModel { - id: Set(id_bytes.to_vec()), - owner_type: Set(OwnerType::Single), - frozen: Set(false), - tree_id: Set(Some(tree_id.clone())), - specification_version: Set(Some(SpecificationVersions::V1)), - specification_asset_class: Set(Some(SpecificationAssetClass::Nft)), - nonce: Set(Some(nonce as i64)), - royalty_target_type: Set(RoyaltyTargetType::Creators), - royalty_target: Set(None), - royalty_amount: Set(metadata.seller_fee_basis_points as i32), //basis points - asset_data: Set(Some(id_bytes.to_vec())), - slot_updated: Set(Some(slot_i)), - ..Default::default() - }; - - // Upsert asset table base info. - let mut query = asset::Entity::insert(asset_model) - .on_conflict( - OnConflict::columns([asset::Column::Id]) - .update_columns([ - asset::Column::OwnerType, - asset::Column::Frozen, - asset::Column::SpecificationVersion, - asset::Column::SpecificationAssetClass, - asset::Column::RoyaltyTargetType, - asset::Column::RoyaltyTarget, - asset::Column::RoyaltyAmount, - asset::Column::AssetData, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - - // Do not overwrite changes that happened after the asset was decompressed. - query.sql = format!( - "{} WHERE excluded.slot_updated > asset.slot_updated OR asset.slot_updated IS NULL", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - - // Partial update of asset table with just compression info elements. - upsert_asset_with_compression_info( - txn, - id_bytes.to_vec(), - true, - false, - 1, - None, - false, - ) - .await?; - - // Partial update of asset table with just leaf. - upsert_asset_with_leaf_info( - txn, - id_bytes.to_vec(), - nonce as i64, - tree_id, - le.leaf_hash.to_vec(), - le.schema.data_hash(), - le.schema.creator_hash(), - seq as i64, - false, - ) - .await?; - - // Partial update of asset table with just leaf owner and delegate. - upsert_asset_with_owner_and_delegate_info( - txn, - id_bytes.to_vec(), - owner.to_bytes().to_vec(), - delegate, - seq as i64, - ) - .await?; - - upsert_asset_with_seq(txn, id_bytes.to_vec(), seq as i64).await?; - - let attachment = asset_v1_account_attachments::ActiveModel { - id: Set(edition_attachment_address.to_bytes().to_vec()), - slot_updated: Set(slot_i), - attachment_type: Set(V1AccountAttachments::MasterEditionV2), - ..Default::default() - }; - - let query = asset_v1_account_attachments::Entity::insert(attachment) - .on_conflict( - OnConflict::columns([asset_v1_account_attachments::Column::Id]) - .do_nothing() - .to_owned(), - ) - .build(DbBackend::Postgres); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - - // Insert into `asset_creators` table. - let creators = &metadata.creators; - if !creators.is_empty() { - // Vec to hold base creator information. - let mut db_creator_infos = Vec::with_capacity(creators.len()); - - // Vec to hold info on whether a creator is verified. This info is protected by `seq` number. - let mut db_creator_verified_infos = Vec::with_capacity(creators.len()); - - // Set to prevent duplicates. - let mut creators_set = HashSet::new(); - - for (i, c) in creators.iter().enumerate() { - if creators_set.contains(&c.address) { - continue; - } - db_creator_infos.push(asset_creators::ActiveModel { - asset_id: Set(id_bytes.to_vec()), - creator: Set(c.address.to_bytes().to_vec()), - position: Set(i as i16), - share: Set(c.share as i32), - slot_updated: Set(Some(slot_i)), - ..Default::default() - }); - - db_creator_verified_infos.push(asset_creators::ActiveModel { - asset_id: Set(id_bytes.to_vec()), - creator: Set(c.address.to_bytes().to_vec()), - verified: Set(c.verified), - seq: Set(Some(seq as i64)), - ..Default::default() - }); - - creators_set.insert(c.address); - } - - // This statement will update base information for each creator. - let query = asset_creators::Entity::insert_many(db_creator_infos) - .on_conflict( - OnConflict::columns([ - asset_creators::Column::AssetId, - asset_creators::Column::Creator, - ]) - .update_columns([ - asset_creators::Column::Position, - asset_creators::Column::Share, - asset_creators::Column::SlotUpdated, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - txn.execute(query).await?; - - // This statement will update whether the creator is verified and the `seq` - // number. `seq` is used to protect the `verified` field, allowing for `mint` - // and `verifyCreator` to be processed out of order. - let mut query = asset_creators::Entity::insert_many(db_creator_verified_infos) - .on_conflict( - OnConflict::columns([ - asset_creators::Column::AssetId, - asset_creators::Column::Creator, - ]) - .update_columns([ - asset_creators::Column::Verified, - asset_creators::Column::Seq, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.seq > asset_creators.seq OR asset_creators.seq IS NULL", - query.sql - ); - txn.execute(query).await?; - } - - // Insert into `asset_authority` table. - let model = asset_authority::ActiveModel { - asset_id: Set(id_bytes.to_vec()), - authority: Set(bundle.keys.get(0).unwrap().0.to_vec()), //TODO - we need to rem,ove the optional bubblegum signer logic - seq: Set(seq as i64), - slot_updated: Set(slot_i), - ..Default::default() - }; - - // Do not attempt to modify any existing values: - // `ON CONFLICT ('asset_id') DO NOTHING`. - let query = asset_authority::Entity::insert(model) - .on_conflict( - OnConflict::columns([asset_authority::Column::AssetId]) - .do_nothing() - .to_owned(), - ) - .build(DbBackend::Postgres); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - - // Upsert into `asset_grouping` table with base collection info. - upsert_collection_info( - txn, - id_bytes.to_vec(), - metadata.collection.clone(), - slot_i, - seq as i64, - ) - .await?; - - if uri.is_empty() { - warn!( - "URI is empty for mint {}. Skipping background task.", - bs58::encode(id).into_string() - ); - return Ok(None); - } - - let mut task = DownloadMetadata { - asset_data_id: id_bytes.to_vec(), - uri: metadata.uri.clone(), - created_at: Some(Utc::now().naive_utc()), - }; - task.sanitize(); - let t = task.into_task_data()?; - Ok(Some(t)) - } - _ => Err(IngesterError::NotImplemented), - }; - } - Err(IngesterError::ParsingError( - "Ix not parsed correctly".to_string(), - )) -} diff --git a/nft_ingester/src/program_transformers/mod.rs b/nft_ingester/src/program_transformers/mod.rs deleted file mode 100644 index fc555829d..000000000 --- a/nft_ingester/src/program_transformers/mod.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::{error::IngesterError, tasks::TaskData}; -use blockbuster::{ - instruction::{order_instructions, InstructionBundle, IxPair}, - program_handler::ProgramParser, - programs::{ - bubblegum::BubblegumParser, token_account::TokenAccountParser, - token_metadata::TokenMetadataParser, ProgramParseResult, - }, -}; -use log::{debug, error, info}; -use plerkle_serialization::{AccountInfo, Pubkey as FBPubkey, TransactionInfo}; -use sea_orm::{DatabaseConnection, SqlxPostgresConnector}; -use solana_sdk::pubkey::Pubkey; -use sqlx::PgPool; -use std::collections::{HashMap, HashSet, VecDeque}; -use tokio::sync::mpsc::UnboundedSender; - -use crate::program_transformers::{ - bubblegum::handle_bubblegum_instruction, token::handle_token_program_account, - token_metadata::handle_token_metadata_account, -}; - -mod bubblegum; -mod token; -mod token_metadata; - -pub struct ProgramTransformer { - storage: DatabaseConnection, - task_sender: UnboundedSender, - matchers: HashMap>, - key_set: HashSet, - cl_audits: bool, -} - -impl ProgramTransformer { - pub fn new(pool: PgPool, task_sender: UnboundedSender, cl_audits: bool) -> Self { - let mut matchers: HashMap> = HashMap::with_capacity(1); - let bgum = BubblegumParser {}; - let token_metadata = TokenMetadataParser {}; - let token = TokenAccountParser {}; - matchers.insert(bgum.key(), Box::new(bgum)); - matchers.insert(token_metadata.key(), Box::new(token_metadata)); - matchers.insert(token.key(), Box::new(token)); - let hs = matchers.iter().fold(HashSet::new(), |mut acc, (k, _)| { - acc.insert(*k); - acc - }); - let pool: PgPool = pool; - ProgramTransformer { - storage: SqlxPostgresConnector::from_sqlx_postgres_pool(pool), - task_sender, - matchers, - key_set: hs, - cl_audits, - } - } - - pub fn break_transaction<'i>( - &self, - tx: &'i TransactionInfo<'i>, - ) -> VecDeque<(IxPair<'i>, Option>>)> { - let ref_set: HashSet<&[u8]> = self.key_set.iter().map(|k| k.as_ref()).collect(); - order_instructions(ref_set, tx) - } - - #[allow(clippy::borrowed_box)] - pub fn match_program(&self, key: &FBPubkey) -> Option<&Box> { - self.matchers - .get(&Pubkey::try_from(key.0.as_slice()).expect("valid key from FlatBuffer")) - } - - pub async fn handle_transaction<'a>( - &self, - tx: &'a TransactionInfo<'a>, - ) -> Result<(), IngesterError> { - let sig: Option<&str> = tx.signature(); - info!("Handling Transaction: {:?}", sig); - let instructions = self.break_transaction(tx); - let accounts = tx.account_keys().unwrap_or_default(); - let slot = tx.slot(); - let txn_id = tx.signature().unwrap_or(""); - let mut keys: Vec = Vec::with_capacity(accounts.len()); - for k in accounts.into_iter() { - keys.push(*k); - } - let mut not_impl = 0; - let ixlen = instructions.len(); - debug!("Instructions: {}", ixlen); - let contains = instructions - .iter() - .filter(|(ib, _inner)| ib.0 .0.as_ref() == mpl_bubblegum::ID.as_ref()); - debug!("Instructions bgum: {}", contains.count()); - for (outer_ix, inner_ix) in instructions { - let (program, instruction) = outer_ix; - let ix_accounts = instruction.accounts().unwrap().iter().collect::>(); - let ix_account_len = ix_accounts.len(); - let max = ix_accounts.iter().max().copied().unwrap_or(0) as usize; - if keys.len() < max { - return Err(IngesterError::DeserializationError( - "Missing Accounts in Serialized Ixn/Txn".to_string(), - )); - } - let ix_accounts = - ix_accounts - .iter() - .fold(Vec::with_capacity(ix_account_len), |mut acc, a| { - if let Some(key) = keys.get(*a as usize) { - acc.push(*key); - } - acc - }); - let ix = InstructionBundle { - txn_id, - program, - instruction: Some(instruction), - inner_ix, - keys: ix_accounts.as_slice(), - slot, - }; - - if let Some(program) = self.match_program(&ix.program) { - debug!("Found a ix for program: {:?}", program.key()); - let result = program.handle_instruction(&ix)?; - let concrete = result.result_type(); - match concrete { - ProgramParseResult::Bubblegum(parsing_result) => { - handle_bubblegum_instruction( - parsing_result, - &ix, - &self.storage, - &self.task_sender, - self.cl_audits, - ) - .await - .map_err(|err| { - error!( - "Failed to handle bubblegum instruction for txn {:?}: {:?}", - sig, err - ); - err - })?; - } - _ => { - not_impl += 1; - } - }; - } - } - - if not_impl == ixlen { - debug!("Not imple"); - return Err(IngesterError::NotImplemented); - } - Ok(()) - } - - pub async fn handle_account_update<'b>( - &self, - acct: AccountInfo<'b>, - ) -> Result<(), IngesterError> { - let owner = acct.owner().unwrap(); - if let Some(program) = self.match_program(owner) { - let result = program.handle_account(&acct)?; - let concrete = result.result_type(); - match concrete { - ProgramParseResult::TokenMetadata(parsing_result) => { - handle_token_metadata_account( - &acct, - parsing_result, - &self.storage, - &self.task_sender, - ) - .await - } - ProgramParseResult::TokenProgramAccount(parsing_result) => { - handle_token_program_account( - &acct, - parsing_result, - &self.storage, - &self.task_sender, - ) - .await - } - _ => Err(IngesterError::NotImplemented), - }?; - } - Ok(()) - } -} diff --git a/nft_ingester/src/program_transformers/token/mod.rs b/nft_ingester/src/program_transformers/token/mod.rs deleted file mode 100644 index c32cad020..000000000 --- a/nft_ingester/src/program_transformers/token/mod.rs +++ /dev/null @@ -1,141 +0,0 @@ -use crate::{error::IngesterError, tasks::TaskData}; -use blockbuster::programs::token_account::TokenProgramAccount; -use digital_asset_types::dao::{asset, token_accounts, tokens}; -use plerkle_serialization::AccountInfo; -use sea_orm::{ - entity::*, query::*, sea_query::OnConflict, ActiveValue::Set, ConnectionTrait, - DatabaseConnection, DbBackend, EntityTrait, -}; -use solana_sdk::program_option::COption; -use spl_token::state::AccountState; -use tokio::sync::mpsc::UnboundedSender; - -pub async fn handle_token_program_account<'a, 'b, 'c>( - account_update: &'a AccountInfo<'a>, - parsing_result: &'b TokenProgramAccount, - db: &'c DatabaseConnection, - _task_manager: &UnboundedSender, -) -> Result<(), IngesterError> { - let key = *account_update.pubkey().unwrap(); - let key_bytes = key.0.to_vec(); - let spl_token_program = account_update.owner().unwrap().0.to_vec(); - match &parsing_result { - TokenProgramAccount::TokenAccount(ta) => { - let mint = ta.mint.to_bytes().to_vec(); - let delegate: Option> = match ta.delegate { - COption::Some(d) => Some(d.to_bytes().to_vec()), - COption::None => None, - }; - let frozen = matches!(ta.state, AccountState::Frozen); - let owner = ta.owner.to_bytes().to_vec(); - let model = token_accounts::ActiveModel { - pubkey: Set(key_bytes), - mint: Set(mint.clone()), - delegate: Set(delegate.clone()), - owner: Set(owner.clone()), - frozen: Set(frozen), - delegated_amount: Set(ta.delegated_amount as i64), - token_program: Set(spl_token_program), - slot_updated: Set(account_update.slot() as i64), - amount: Set(ta.amount as i64), - close_authority: Set(None), - }; - - let mut query = token_accounts::Entity::insert(model) - .on_conflict( - OnConflict::columns([token_accounts::Column::Pubkey]) - .update_columns([ - token_accounts::Column::Mint, - token_accounts::Column::DelegatedAmount, - token_accounts::Column::Delegate, - token_accounts::Column::Amount, - token_accounts::Column::Frozen, - token_accounts::Column::TokenProgram, - token_accounts::Column::Owner, - token_accounts::Column::CloseAuthority, - token_accounts::Column::SlotUpdated, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > token_accounts.slot_updated", - query.sql - ); - db.execute(query).await?; - let txn = db.begin().await?; - let asset_update: Option = asset::Entity::find_by_id(mint) - .filter(asset::Column::OwnerType.eq("single")) - .one(&txn) - .await?; - if let Some(asset) = asset_update { - // will only update owner if token account balance is non-zero - if ta.amount > 0 { - let mut active: asset::ActiveModel = asset.into(); - active.owner = Set(Some(owner)); - active.delegate = Set(delegate); - active.frozen = Set(frozen); - active.save(&txn).await?; - } - } - txn.commit().await?; - Ok(()) - } - TokenProgramAccount::Mint(m) => { - let freeze_auth: Option> = match m.freeze_authority { - COption::Some(d) => Some(d.to_bytes().to_vec()), - COption::None => None, - }; - let mint_auth: Option> = match m.mint_authority { - COption::Some(d) => Some(d.to_bytes().to_vec()), - COption::None => None, - }; - let model = tokens::ActiveModel { - mint: Set(key_bytes.clone()), - token_program: Set(spl_token_program), - slot_updated: Set(account_update.slot() as i64), - supply: Set(m.supply as i64), - decimals: Set(m.decimals as i32), - close_authority: Set(None), - extension_data: Set(None), - mint_authority: Set(mint_auth), - freeze_authority: Set(freeze_auth), - }; - - let mut query = tokens::Entity::insert(model) - .on_conflict( - OnConflict::columns([tokens::Column::Mint]) - .update_columns([ - tokens::Column::Supply, - tokens::Column::TokenProgram, - tokens::Column::MintAuthority, - tokens::Column::CloseAuthority, - tokens::Column::ExtensionData, - tokens::Column::SlotUpdated, - tokens::Column::Decimals, - tokens::Column::FreezeAuthority, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > tokens.slot_updated", - query.sql - ); - db.execute(query).await?; - let asset_update: Option = asset::Entity::find_by_id(key_bytes.clone()) - .filter(asset::Column::OwnerType.eq("single")) - .one(db) - .await?; - if let Some(asset) = asset_update { - let mut active: asset::ActiveModel = asset.into(); - active.supply = Set(m.supply as i64); - active.supply_mint = Set(Some(key_bytes)); - active.save(db).await?; - } - Ok(()) - } - _ => Err(IngesterError::NotImplemented), - }?; - Ok(()) -} diff --git a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs deleted file mode 100644 index 4f81a4636..000000000 --- a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs +++ /dev/null @@ -1,393 +0,0 @@ -use crate::{error::IngesterError, tasks::TaskData}; -use blockbuster::token_metadata::{ - pda::find_master_edition_account, - state::{Metadata, TokenStandard, UseMethod, Uses}, -}; -use chrono::Utc; -use digital_asset_types::{ - dao::{ - asset, asset_authority, asset_creators, asset_data, asset_grouping, - asset_v1_account_attachments, - sea_orm_active_enums::{ - ChainMutability, Mutability, OwnerType, RoyaltyTargetType, SpecificationAssetClass, - SpecificationVersions, V1AccountAttachments, - }, - token_accounts, tokens, - }, - json::ChainDataV1, -}; - -use crate::tasks::{DownloadMetadata, IntoTaskData}; -use log::warn; -use num_traits::FromPrimitive; -use plerkle_serialization::Pubkey as FBPubkey; -use sea_orm::{ - entity::*, query::*, sea_query::OnConflict, ActiveValue::Set, ConnectionTrait, DbBackend, - DbErr, EntityTrait, JsonValue, -}; -use std::collections::HashSet; - -pub async fn burn_v1_asset( - conn: &T, - id: FBPubkey, - slot: u64, -) -> Result<(), IngesterError> { - let (id, slot_i) = (id.0, slot as i64); - let model = asset::ActiveModel { - id: Set(id.to_vec()), - slot_updated: Set(Some(slot_i)), - burnt: Set(true), - ..Default::default() - }; - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::columns([asset::Column::Id]) - .update_columns([asset::Column::SlotUpdated, asset::Column::Burnt]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset.slot_updated", - query.sql - ); - conn.execute(query).await?; - Ok(()) -} - -pub async fn save_v1_asset( - conn: &T, - id: FBPubkey, - slot: u64, - metadata: &Metadata, -) -> Result, IngesterError> { - let metadata = metadata.clone(); - let data = metadata.data; - let meta_mint_pubkey = metadata.mint; - let (edition_attachment_address, _) = find_master_edition_account(&meta_mint_pubkey); - let mint = metadata.mint.to_bytes().to_vec(); - let authority = metadata.update_authority.to_bytes().to_vec(); - let id = id.0; - let slot_i = slot as i64; - let uri = data.uri.trim().replace('\0', ""); - let _spec = SpecificationVersions::V1; - let class = match metadata.token_standard { - Some(TokenStandard::NonFungible) => SpecificationAssetClass::Nft, - Some(TokenStandard::FungibleAsset) => SpecificationAssetClass::FungibleAsset, - Some(TokenStandard::Fungible) => SpecificationAssetClass::FungibleToken, - _ => SpecificationAssetClass::Unknown, - }; - let ownership_type = match class { - SpecificationAssetClass::FungibleAsset => OwnerType::Token, - SpecificationAssetClass::FungibleToken => OwnerType::Token, - _ => OwnerType::Single, - }; - - // gets the token and token account for the mint to populate the asset. This is required when the token and token account are indexed, but not the metadata account. If the metadata account is indexed, then the token and ta ingester will update the asset with the correct data - - let (token, token_account): (Option, Option) = - match ownership_type { - OwnerType::Single => { - let token: Option = - tokens::Entity::find_by_id(mint.clone()).one(conn).await?; - // query for token account associated with mint with positive balance - let token_account: Option = token_accounts::Entity::find() - .filter(token_accounts::Column::Mint.eq(mint.clone())) - .filter(token_accounts::Column::Amount.gt(0)) - .one(conn) - .await?; - Ok((token, token_account)) - } - _ => { - let token = tokens::Entity::find_by_id(mint.clone()).one(conn).await?; - Ok((token, None)) - } - } - .map_err(|e: DbErr| IngesterError::DatabaseError(e.to_string()))?; - - // get supply of token, default to 1 since most cases will be NFTs. Token mint ingester will properly set supply if token_result is None - let (supply, supply_mint) = match token { - Some(t) => (Set(t.supply), Set(Some(t.mint))), - None => (Set(1), NotSet), - }; - - // owner and delegate should be from the token account with the mint - let (owner, delegate) = match token_account { - Some(ta) => (Set(Some(ta.owner)), Set(ta.delegate)), - None => (NotSet, NotSet), - }; - - let name = data.name.clone().into_bytes(); - let symbol = data.symbol.clone().into_bytes(); - - let mut chain_data = ChainDataV1 { - name: data.name.clone(), - symbol: data.symbol.clone(), - edition_nonce: metadata.edition_nonce, - primary_sale_happened: metadata.primary_sale_happened, - token_standard: metadata.token_standard, - uses: metadata.uses.map(|u| Uses { - use_method: UseMethod::from_u8(u.use_method as u8).unwrap(), - remaining: u.remaining, - total: u.total, - }), - }; - chain_data.sanitize(); - let chain_data_json = serde_json::to_value(chain_data) - .map_err(|e| IngesterError::DeserializationError(e.to_string()))?; - let chain_mutability = match metadata.is_mutable { - true => ChainMutability::Mutable, - false => ChainMutability::Immutable, - }; - let asset_data_model = asset_data::ActiveModel { - chain_data_mutability: Set(chain_mutability), - chain_data: Set(chain_data_json), - metadata_url: Set(uri.clone()), - metadata: Set(JsonValue::String("processing".to_string())), - metadata_mutability: Set(Mutability::Mutable), - slot_updated: Set(slot_i), - reindex: Set(Some(true)), - id: Set(id.to_vec()), - raw_name: Set(name.to_vec()), - raw_symbol: Set(symbol.to_vec()), - }; - let txn = conn.begin().await?; - let mut query = asset_data::Entity::insert(asset_data_model) - .on_conflict( - OnConflict::columns([asset_data::Column::Id]) - .update_columns([ - asset_data::Column::ChainDataMutability, - asset_data::Column::ChainData, - asset_data::Column::MetadataUrl, - asset_data::Column::MetadataMutability, - asset_data::Column::SlotUpdated, - asset_data::Column::Reindex, - asset_data::Column::RawName, - asset_data::Column::RawSymbol, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset_data.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - let model = asset::ActiveModel { - id: Set(id.to_vec()), - owner, - owner_type: Set(ownership_type), - delegate, - frozen: Set(false), - supply, - supply_mint, - specification_version: Set(Some(SpecificationVersions::V1)), - specification_asset_class: Set(Some(class)), - tree_id: Set(None), - nonce: Set(Some(0)), - seq: Set(Some(0)), - leaf: Set(None), - compressed: Set(false), - compressible: Set(false), - royalty_target_type: Set(RoyaltyTargetType::Creators), - royalty_target: Set(None), - royalty_amount: Set(data.seller_fee_basis_points as i32), //basis points - asset_data: Set(Some(id.to_vec())), - slot_updated: Set(Some(slot_i)), - burnt: Set(false), - ..Default::default() - }; - let mut query = asset::Entity::insert(model) - .on_conflict( - OnConflict::columns([asset::Column::Id]) - .update_columns([ - asset::Column::Owner, - asset::Column::OwnerType, - asset::Column::Delegate, - asset::Column::Frozen, - asset::Column::Supply, - asset::Column::SupplyMint, - asset::Column::SpecificationVersion, - asset::Column::SpecificationAssetClass, - asset::Column::TreeId, - asset::Column::Nonce, - asset::Column::Seq, - asset::Column::Leaf, - asset::Column::Compressed, - asset::Column::Compressible, - asset::Column::RoyaltyTargetType, - asset::Column::RoyaltyTarget, - asset::Column::RoyaltyAmount, - asset::Column::AssetData, - asset::Column::SlotUpdated, - asset::Column::Burnt, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - let attachment = asset_v1_account_attachments::ActiveModel { - id: Set(edition_attachment_address.to_bytes().to_vec()), - slot_updated: Set(slot_i), - attachment_type: Set(V1AccountAttachments::MasterEditionV2), - ..Default::default() - }; - let query = asset_v1_account_attachments::Entity::insert(attachment) - .on_conflict( - OnConflict::columns([asset_v1_account_attachments::Column::Id]) - .do_nothing() - .to_owned(), - ) - .build(DbBackend::Postgres); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - let model = asset_authority::ActiveModel { - asset_id: Set(id.to_vec()), - authority: Set(authority), - seq: Set(0), - slot_updated: Set(slot_i), - ..Default::default() - }; - let mut query = asset_authority::Entity::insert(model) - .on_conflict( - OnConflict::columns([asset_authority::Column::AssetId]) - .update_columns([ - asset_authority::Column::Authority, - asset_authority::Column::Seq, - asset_authority::Column::SlotUpdated, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset_authority.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - if let Some(c) = &metadata.collection { - let model = asset_grouping::ActiveModel { - asset_id: Set(id.to_vec()), - group_key: Set("collection".to_string()), - group_value: Set(Some(c.key.to_string())), - verified: Set(Some(c.verified)), - seq: Set(None), - slot_updated: Set(Some(slot_i)), - ..Default::default() - }; - let mut query = asset_grouping::Entity::insert(model) - .on_conflict( - OnConflict::columns([ - asset_grouping::Column::AssetId, - asset_grouping::Column::GroupKey, - ]) - .update_columns([ - asset_grouping::Column::GroupKey, - asset_grouping::Column::GroupValue, - asset_grouping::Column::Seq, - asset_grouping::Column::SlotUpdated, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset_grouping.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - } - txn.commit().await?; - let creators = data.creators.unwrap_or_default(); - if !creators.is_empty() { - let mut creators_set = HashSet::new(); - let existing_creators: Vec = asset_creators::Entity::find() - .filter( - Condition::all() - .add(asset_creators::Column::AssetId.eq(id.to_vec())) - .add(asset_creators::Column::SlotUpdated.lt(slot_i)), - ) - .all(conn) - .await?; - if !existing_creators.is_empty() { - let mut db_creators = Vec::with_capacity(creators.len()); - for (i, c) in creators.into_iter().enumerate() { - if creators_set.contains(&c.address) { - continue; - } - db_creators.push(asset_creators::ActiveModel { - asset_id: Set(id.to_vec()), - creator: Set(c.address.to_bytes().to_vec()), - share: Set(c.share as i32), - verified: Set(c.verified), - seq: Set(Some(0)), - slot_updated: Set(Some(slot_i)), - position: Set(i as i16), - ..Default::default() - }); - creators_set.insert(c.address); - } - let txn = conn.begin().await?; - asset_creators::Entity::delete_many() - .filter( - Condition::all() - .add(asset_creators::Column::AssetId.eq(id.to_vec())) - .add(asset_creators::Column::SlotUpdated.lt(slot_i)), - ) - .exec(&txn) - .await?; - if !db_creators.is_empty() { - let mut query = asset_creators::Entity::insert_many(db_creators) - .on_conflict( - OnConflict::columns([ - asset_creators::Column::AssetId, - asset_creators::Column::Position, - ]) - .update_columns([ - asset_creators::Column::Creator, - asset_creators::Column::Share, - asset_creators::Column::Verified, - asset_creators::Column::Seq, - asset_creators::Column::SlotUpdated, - ]) - .to_owned(), - ) - .build(DbBackend::Postgres); - query.sql = format!( - "{} WHERE excluded.slot_updated > asset_creators.slot_updated", - query.sql - ); - txn.execute(query) - .await - .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?; - } - txn.commit().await?; - } - } - if uri.is_empty() { - warn!( - "URI is empty for mint {}. Skipping background task.", - bs58::encode(mint).into_string() - ); - return Ok(None); - } - - let mut task = DownloadMetadata { - asset_data_id: id.to_vec(), - uri, - created_at: Some(Utc::now().naive_utc()), - }; - task.sanitize(); - let t = task.into_task_data()?; - Ok(Some(t)) -} From 83b171eb23a7cab46d007468890f274abd59cab2 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 23 Nov 2023 11:02:05 -0500 Subject: [PATCH 10/72] remove not used deps --- Cargo.lock | 1690 +++++++++---------- das_api/Cargo.toml | 28 +- digital_asset_types/Cargo.toml | 14 +- digital_asset_types/src/dao/scopes/asset.rs | 10 +- metaplex-rpc-proxy/Cargo.toml | 2 - nft_ingester/Cargo.toml | 1 - nft_ingester/src/backfiller.rs | 2 +- tools/bgtask_creator/Cargo.toml | 22 +- tools/load_generation/Cargo.toml | 4 +- tools/tree-status/Cargo.toml | 31 +- tools/txn_forwarder/Cargo.toml | 8 +- 11 files changed, 792 insertions(+), 1020 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a5e4dd91..8a629c27a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,8 +18,8 @@ version = "0.7.12" dependencies = [ "anyhow", "bs58 0.4.0", - "clap 4.5.4", - "env_logger 0.10.2", + "clap 4.4.8", + "env_logger 0.10.0", "figment", "flatbuffers", "futures", @@ -93,11 +93,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.10", "once_cell", "version_check", ] @@ -109,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.10", "once_cell", "version_check", "zerocopy", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f2a3e1df4685f18d12a943a9f2a7456305401af21a07c9fe076ef9ecd6e400" dependencies = [ "anchor-syn", - "bs58 0.5.1", + "bs58 0.5.0", "proc-macro2", "quote", "syn 1.0.109", @@ -295,7 +295,7 @@ dependencies = [ "bincode", "borsh 0.10.3", "bytemuck", - "getrandom 0.2.14", + "getrandom 0.2.10", "solana-program", "thiserror", ] @@ -307,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9101b84702fed2ea57bd22992f75065da5648017135b844283a2f6d74f27825" dependencies = [ "anyhow", - "bs58 0.5.1", + "bs58 0.5.0", "heck 0.3.3", "proc-macro2", "quote", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -358,49 +358,49 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "ark-bn254" @@ -550,7 +550,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.35", + "time 0.3.29", ] [[package]] @@ -605,22 +605,22 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" +checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" dependencies = [ "concurrent-queue", - "event-listener 5.3.0", - "event-listener-strategy 0.5.1", + "event-listener 3.1.0", + "event-listener-strategy", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" dependencies = [ "brotli", "flate2", @@ -632,30 +632,30 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.10.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f98c37cf288e302c16ef6c8472aad1e034c6c84ce5ea7b8101c98eb4a802fee" +checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" dependencies = [ - "async-lock 3.3.0", + "async-lock 3.1.1", "async-task", "concurrent-queue", - "fastrand 2.0.2", - "futures-lite 2.3.0", + "fastrand 2.0.1", + "futures-lite 2.0.1", "slab", ] [[package]] name = "async-global-executor" -version = "2.4.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ - "async-channel 2.2.0", + "async-channel 1.9.0", "async-executor", - "async-io 2.3.2", - "async-lock 3.3.0", + "async-io", + "async-lock 2.8.0", "blocking", - "futures-lite 2.3.0", + "futures-lite 1.13.0", "once_cell", "tokio", ] @@ -673,32 +673,13 @@ dependencies = [ "futures-lite 1.13.0", "log", "parking", - "polling 2.8.0", + "polling", "rustix 0.37.27", "slab", - "socket2 0.4.10", + "socket2 0.4.9", "waker-fn", ] -[[package]] -name = "async-io" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" -dependencies = [ - "async-lock 3.3.0", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.3.0", - "parking", - "polling 3.6.0", - "rustix 0.38.32", - "slab", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "async-lock" version = "2.8.0" @@ -710,12 +691,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.3.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", + "event-listener 3.1.0", + "event-listener-strategy", "pin-project-lite", ] @@ -737,7 +718,7 @@ dependencies = [ "async-attributes", "async-channel 1.9.0", "async-global-executor", - "async-io 1.13.0", + "async-io", "async-lock 2.8.0", "crossbeam-utils", "futures-channel", @@ -774,24 +755,24 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "async-task" -version = "4.7.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -837,17 +818,17 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.2.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backon" -version = "0.4.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" dependencies = [ - "fastrand 2.0.2", + "fastrand 1.9.0", "futures-core", "pin-project", "tokio", @@ -855,9 +836,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -895,9 +876,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.7" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "base64" @@ -925,7 +906,7 @@ name = "bgtask_creator" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.5.4", + "clap 4.4.8", "digital_asset_types", "futures", "lazy_static", @@ -957,9 +938,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" dependencies = [ "serde", ] @@ -987,9 +968,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", @@ -1060,12 +1041,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.2.0", - "async-lock 3.3.0", + "async-channel 2.1.0", + "async-lock 3.1.1", "async-task", - "fastrand 2.0.2", + "fastrand 2.0.1", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.0.1", "piper", "tracing", ] @@ -1090,16 +1071,6 @@ dependencies = [ "hashbrown 0.13.2", ] -[[package]] -name = "borsh" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0901fc8eb0aca4c83be0106d6f2db17d86a08dfc2c25f0e84464bf381158add6" -dependencies = [ - "borsh-derive 1.4.0", - "cfg_aliases", -] - [[package]] name = "borsh-derive" version = "0.9.3" @@ -1126,20 +1097,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "borsh-derive" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51670c3aa053938b0ee3bd67c3817e471e626151131b934038e83c5bf8de48f5" -dependencies = [ - "once_cell", - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 2.0.58", - "syn_derive", -] - [[package]] name = "borsh-derive-internal" version = "0.9.3" @@ -1186,9 +1143,9 @@ dependencies = [ [[package]] name = "brotli" -version = "4.0.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1197,9 +1154,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "3.0.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" +checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1213,18 +1170,18 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" dependencies = [ "tinyvec", ] [[package]] name = "bstr" -version = "1.9.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "serde", @@ -1232,9 +1189,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bv" @@ -1248,9 +1205,9 @@ dependencies = [ [[package]] name = "bytecheck" -version = "0.6.12" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1259,9 +1216,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.12" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ "proc-macro2", "quote", @@ -1279,13 +1236,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.6.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -1296,9 +1253,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cadence" @@ -1330,9 +1287,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.92" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -1344,17 +1301,11 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" - [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1362,7 +1313,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.48.5", ] [[package]] @@ -1403,29 +1354,29 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.1", + "textwrap 0.16.0", ] [[package]] name = "clap" -version = "4.5.4" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", - "clap_derive 4.5.4", + "clap_derive 4.4.7", ] [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", - "clap_lex 0.7.0", - "strsim 0.11.1", + "clap_lex 0.6.0", + "strsim 0.10.0", ] [[package]] @@ -1443,14 +1394,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -1464,9 +1415,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -1489,9 +1440,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.7" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes", "futures-core", @@ -1503,24 +1454,24 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" dependencies = [ "crossbeam-utils", ] [[package]] name = "console" -version = "0.15.8" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.52.0", + "windows-sys 0.45.0", ] [[package]] @@ -1557,9 +1508,9 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -1567,85 +1518,95 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.2.1" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ + "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ + "autocfg", + "cfg-if", "crossbeam-utils", + "memoffset 0.9.0", + "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ + "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] [[package]] name = "crunchy" @@ -1698,9 +1659,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.8" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ "darling_core", "darling_macro", @@ -1708,27 +1669,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -1739,7 +1700,7 @@ dependencies = [ "backon", "cadence", "cadence-macros", - "clap 4.5.4", + "clap 4.4.8", "figment", "plerkle_messenger", "solana-account-decoder", @@ -1761,10 +1722,10 @@ dependencies = [ "borsh 0.10.3", "cadence", "cadence-macros", - "clap 4.5.4", + "clap 4.4.8", "das-core", "digital_asset_types", - "env_logger 0.10.2", + "env_logger 0.10.0", "figment", "flatbuffers", "futures", @@ -1794,7 +1755,7 @@ dependencies = [ "cadence", "cadence-macros", "digital_asset_types", - "env_logger 0.10.2", + "env_logger 0.10.0", "figment", "hyper", "jsonrpsee", @@ -1836,17 +1797,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.2", + "hashbrown 0.14.1", "lock_api", "once_cell", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.8", ] [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "der" @@ -1873,11 +1834,10 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" dependencies = [ - "powerfmt", "serde", ] @@ -1993,7 +1953,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -2016,7 +1976,7 @@ checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -2027,9 +1987,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "eager" @@ -2074,9 +2034,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" dependencies = [ "serde", ] @@ -2089,31 +2049,31 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] [[package]] name = "enum-iterator" -version = "1.5.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" +checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.3.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" +checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -2131,9 +2091,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ "humantime", "is-terminal", @@ -2150,12 +2110,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -2166,20 +2126,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "5.3.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" dependencies = [ "concurrent-queue", "parking", @@ -2188,21 +2137,11 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" +checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" dependencies = [ - "event-listener 5.3.0", + "event-listener 3.1.0", "pin-project-lite", ] @@ -2227,9 +2166,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "feature-probe" @@ -2244,7 +2183,7 @@ dependencies = [ "anyhow", "async-trait", "borsh 0.10.3", - "clap 4.5.4", + "clap 4.4.8", "mpl-bubblegum", "solana-account-decoder", "solana-client", @@ -2255,15 +2194,15 @@ dependencies = [ [[package]] name = "figment" -version = "0.10.16" +version = "0.10.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdefe49ed1057d124dc81a0681c30dd07de56ad96e32adc7b64e8f28eaab31c4" +checksum = "a014ac935975a70ad13a3bff2463b1c1b083b35ae4cb6309cfc59476aa7a181f" dependencies = [ "atomic 0.6.0", "pear", "serde", "serde_yaml", - "toml 0.8.12", + "toml 0.8.8", "uncased", "version_check", ] @@ -2286,9 +2225,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -2317,9 +2256,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -2353,9 +2292,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -2368,9 +2307,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -2378,15 +2317,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -2406,9 +2345,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" @@ -2427,45 +2366,46 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" dependencies = [ - "fastrand 2.0.2", + "fastrand 2.0.1", "futures-core", "futures-io", + "memchr", "parking", "pin-project-lite", ] [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -2521,9 +2461,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "js-sys", @@ -2534,21 +2474,21 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "globset" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" dependencies = [ "aho-corasick", "bstr", + "fnv", "log", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex", ] [[package]] @@ -2576,9 +2516,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.26" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -2586,7 +2526,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.2.6", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -2608,7 +2548,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.8", + "ahash 0.7.6", ] [[package]] @@ -2617,7 +2557,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.8", + "ahash 0.7.6", ] [[package]] @@ -2631,9 +2571,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ "ahash 0.8.5", "allocator-api2", @@ -2645,7 +2585,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.2", + "hashbrown 0.14.1", ] [[package]] @@ -2683,9 +2623,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -2701,9 +2641,9 @@ checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" [[package]] name = "hkdf" -version = "0.12.4" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" dependencies = [ "hmac 0.12.1", ] @@ -2740,9 +2680,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.12" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -2751,9 +2691,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -2786,9 +2726,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -2801,7 +2741,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2810,14 +2750,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.10", + "rustls 0.21.7", "tokio", "tokio-rustls 0.24.1", ] @@ -2837,16 +2777,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows", ] [[package]] @@ -2866,9 +2806,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2903,25 +2843,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.1", "serde", ] [[package]] name = "indicatif" -version = "0.17.8" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", "instant", "number_prefix", - "portable-atomic 1.6.0", + "portable-atomic 1.4.3", "unicode-width", ] @@ -2933,15 +2873,16 @@ checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "insta" -version = "1.38.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eab73f58e59ca6526037208f0e98851159ec1633cf17b6cd2e1f2c3fd5d53cc" +checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" dependencies = [ "console", "lazy_static", "linked-hash-map", "serde", "similar", + "yaml-rust", ] [[package]] @@ -2993,26 +2934,26 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi 0.3.3", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.9.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.9", - "libc", - "windows-sys 0.52.0", + "hermit-abi 0.3.3", + "rustix 0.38.18", + "windows-sys 0.48.0", ] [[package]] @@ -3026,33 +2967,33 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -3173,18 +3114,18 @@ dependencies = [ [[package]] name = "kaigan" -version = "0.2.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e623cca1f0e2a0919032c1bdabbf81dd9aa34658d5066aca7bb90d608317ab91" +checksum = "4a26f49495f94a283312e7ef45a243540ef20c9356bb01c8d84a61ac8ba5339b" dependencies = [ "borsh 0.10.3", ] [[package]] name = "keccak" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" dependencies = [ "cpufeatures", ] @@ -3206,19 +3147,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "libredox" -version = "0.1.3" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.5.0", - "libc", -] +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libsecp256k1" @@ -3294,9 +3225,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "load_generation" @@ -3315,9 +3246,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -3325,9 +3256,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" dependencies = [ "value-bag", ] @@ -3353,9 +3284,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" @@ -3377,9 +3308,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -3404,8 +3335,6 @@ dependencies = [ "log", "proxy-wasm", "regex", - "wasi 0.7.0", - "wasm-bindgen", ] [[package]] @@ -3414,7 +3343,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ - "ahash 0.7.8", + "ahash 0.7.6", "metrics-macros", "portable-atomic 0.3.20", ] @@ -3464,18 +3393,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.11" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3507,22 +3436,22 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.7.0", + "serde_with 3.6.1", "solana-program", "thiserror", ] [[package]] name = "mpl-token-metadata" -version = "4.1.2" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf0f61b553e424a6234af1268456972ee66c2222e1da89079242251fa7479e5" +checksum = "b4b2de608098eb2ef2a5392069dea83084967e25a4d69d0380a6bb02454fc0fe" dependencies = [ "borsh 0.10.3", "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.7.0", + "serde_with 3.6.1", "solana-program", "thiserror", ] @@ -3550,13 +3479,12 @@ name = "nft_ingester" version = "0.7.2" dependencies = [ "async-trait", - "blockbuster", "borsh 0.10.3", "bs58 0.4.0", "cadence", "cadence-macros", "chrono", - "clap 4.5.4", + "clap 4.4.8", "digital_asset_types", "figment", "flatbuffers", @@ -3664,12 +3592,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-derive" version = "0.3.3" @@ -3683,29 +3605,30 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ + "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.44" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ "autocfg", "num-integer", @@ -3726,9 +3649,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -3739,7 +3662,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi 0.3.3", "libc", ] @@ -3770,7 +3693,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -3779,10 +3702,10 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -3793,9 +3716,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.2" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -3817,9 +3740,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "open-rpc-derive" @@ -3847,11 +3770,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -3868,7 +3791,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -3879,9 +3802,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -3948,7 +3871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.8", ] [[package]] @@ -3967,13 +3890,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.3.5", "smallvec", "windows-targets 0.48.5", ] @@ -4004,9 +3927,9 @@ dependencies = [ [[package]] name = "pear" -version = "0.2.9" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +checksum = "61a386cd715229d399604b50d1361683fe687066f42d56f54be995bc6868f71c" dependencies = [ "inlinable_string", "pear_codegen", @@ -4015,14 +3938,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.9" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +checksum = "da9f0f13dac8069c139e8300a6510e3f4143ecf5259c60b116a9b271b4ca0d54" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -4036,9 +3959,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "percentage" @@ -4051,29 +3974,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -4088,7 +4011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.0.2", + "fastrand 2.0.1", "futures-io", ] @@ -4105,9 +4028,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "plain" @@ -4117,9 +4040,9 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plerkle_messenger" -version = "1.8.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c75e3178528c45bca0b7d515bdc6a3733b5f67d5b37b8a085cf6d7f81e3f2" +checksum = "e2f8e4e8975dcd2dbf94c7f84409096f0a0e5af287eabc9a212704e9e325ec84" dependencies = [ "async-mutex", "async-trait", @@ -4165,21 +4088,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "polling" -version = "3.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.3.9", - "pin-project-lite", - "rustix 0.38.32", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "polyval" version = "0.5.3" @@ -4198,20 +4106,14 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" dependencies = [ - "portable-atomic 1.6.0", + "portable-atomic 1.4.3", ] [[package]] name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "powerfmt" -version = "0.2.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "ppv-lite86" @@ -4238,15 +4140,6 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "proc-macro-crate" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" -dependencies = [ - "toml_edit 0.21.1", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4273,9 +4166,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -4288,7 +4181,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", "version_check", "yansi", ] @@ -4386,7 +4279,7 @@ checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -4400,7 +4293,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.10", + "rustls 0.21.7", "thiserror", "tokio", "tracing", @@ -4416,7 +4309,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.10", + "rustls 0.21.7", "rustls-native-certs", "slab", "thiserror", @@ -4432,16 +4325,16 @@ checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ "bytes", "libc", - "socket2 0.5.6", + "socket2 0.5.4", "tracing", "windows-sys 0.48.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -4549,7 +4442,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.10", ] [[package]] @@ -4572,9 +4465,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -4582,9 +4475,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -4598,7 +4491,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.35", + "time 0.3.29", "yasna", ] @@ -4620,7 +4513,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes", - "combine 4.6.7", + "combine 4.6.6", "futures", "futures-util", "itoa", @@ -4646,34 +4539,34 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.14", - "libredox", + "getrandom 0.2.10", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.3.9", + "regex-syntax 0.7.5", ] [[package]] @@ -4687,13 +4580,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.7.5", ] [[package]] @@ -4704,27 +4597,27 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rend" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" dependencies = [ "bytecheck", ] [[package]] name = "reqwest" -version = "0.11.27" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "async-compression", - "base64 0.21.7", + "base64 0.21.4", "bytes", "encoding_rs", "futures-core", @@ -4743,12 +4636,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.10", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -4759,7 +4651,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.4", + "webpki-roots 0.25.2", "winreg", ] @@ -4780,28 +4672,26 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "911b295d2d302948838c8ac142da1ee09fa7863163b44e6715bc9357905878b8" dependencies = [ "cc", - "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.10", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "rkyv" -version = "0.7.44" +version = "0.7.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" +checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" dependencies = [ "bitvec", "bytecheck", - "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -4813,9 +4703,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.44" +version = "0.7.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" +checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" dependencies = [ "proc-macro2", "quote", @@ -4858,12 +4748,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.35.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" +checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" dependencies = [ "arrayvec", - "borsh 1.4.0", + "borsh 0.10.3", "bytes", "num-traits", "rand 0.8.5", @@ -4886,9 +4776,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-serialize" -version = "0.3.25" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" +checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" [[package]] name = "rustc_version" @@ -4924,15 +4814,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.13", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.10", + "windows-sys 0.48.0", ] [[package]] @@ -4949,12 +4839,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", - "ring 0.17.8", + "ring 0.16.20", "rustls-webpki", "sct", ] @@ -4973,49 +4863,49 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.7", + "base64 0.21.4", ] [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "schemars" -version = "0.8.16" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" dependencies = [ "dyn-clone", "schemars_derive", @@ -5025,9 +4915,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.16" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ "proc-macro2", "quote", @@ -5058,17 +4948,17 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "sct" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5093,7 +4983,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", - "time 0.3.35", + "time 0.3.29", "tracing", "url", "uuid", @@ -5154,7 +5044,7 @@ dependencies = [ "rust_decimal", "sea-query-derive 0.2.0", "serde_json", - "time 0.3.35", + "time 0.3.29", "uuid", ] @@ -5178,7 +5068,7 @@ dependencies = [ "sea-query 0.27.2", "serde_json", "sqlx", - "time 0.3.35", + "time 0.3.29", "uuid", ] @@ -5261,9 +5151,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.10.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -5274,9 +5164,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -5284,37 +5174,37 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.14" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -5330,11 +5220,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -5342,9 +5232,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -5373,20 +5263,20 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.7.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" +checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" dependencies = [ - "base64 0.21.7", + "base64 0.21.4", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.0.2", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.7.0", - "time 0.3.35", + "serde_with_macros 3.6.1", + "time 0.3.29", ] [[package]] @@ -5398,28 +5288,28 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "serde_with_macros" -version = "3.7.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" +checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "serde_yaml" -version = "0.9.34+deprecated" +version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -5448,7 +5338,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -5565,9 +5455,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "similar" -version = "2.5.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" [[package]] name = "sized-chunks" @@ -5590,15 +5480,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -5606,12 +5496,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -5632,12 +5522,12 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d145d4e1e33bfecd209059a0c4c75d623dbcaeb565b4c197f1815257be45726a" +checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" dependencies = [ "Inflector", - "base64 0.21.7", + "base64 0.21.4", "bincode", "bs58 0.4.0", "bv", @@ -5657,9 +5547,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1deaf83f98be3ba9ecee057efa5cdfa6112267e5b1ff53c4ef4b727f66090b9a" +checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" dependencies = [ "chrono", "clap 2.34.0", @@ -5674,16 +5564,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8912026a203ff0d90973e7363f141c6ce569484e06ee0a6f72992144263136" +checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" dependencies = [ "async-trait", "bincode", "dashmap 4.0.2", "futures", "futures-util", - "indexmap 2.2.6", + "indexmap 2.0.2", "indicatif", "log", "quinn", @@ -5707,9 +5597,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4000f4717f86c5f9e1105378e3a6521db770d0ad68417f59960ca4b51103fcd0" +checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" dependencies = [ "bincode", "chrono", @@ -5721,15 +5611,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b1a4d67c01e5c4464ed9bffb656dec2c88e9bfd7d45bf754f650150e5d81db" +checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" dependencies = [ "async-trait", "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.2.6", + "indexmap 2.0.2", "log", "rand 0.8.5", "rayon", @@ -5743,9 +5633,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1790013c7969353000c22907fc21610adb3389a7c9a27a386ebe7fb32b2ad307" +checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" dependencies = [ "ahash 0.8.5", "blake3", @@ -5773,21 +5663,21 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ed2b49a3dd03ddd5107d6e629e8e5895724227a057b3511bf0c107c6d48308" +checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "solana-logger" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc0d5b4f046d07e845b69178989a6b3bf168a82eeee006adb77391b339bce64" +checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5796,9 +5686,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857178177c6b378bcfc35df6867a6eef211059f5e4ab01ee87355d6b7493b556" +checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" dependencies = [ "log", "solana-sdk", @@ -5806,9 +5696,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c68f5cbfbafd002b4d94728748f632a3bd27772ca5c7139710d65940c95477c" +checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" dependencies = [ "crossbeam-channel", "gethostname", @@ -5821,9 +5711,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce93c50199f077df9d8fd4a82bbdfe654422e1d21e5eecb7f878c4e0078f92f" +checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" dependencies = [ "bincode", "clap 3.2.25", @@ -5833,7 +5723,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_derive", - "socket2 0.5.6", + "socket2 0.5.4", "solana-logger", "solana-sdk", "solana-version", @@ -5843,9 +5733,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a233bc2032f4b3f462fe13325cd1a336f355e1d9e793faada488db409cec0d6" +checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" dependencies = [ "ahash 0.8.5", "bincode", @@ -5872,17 +5762,17 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2ae4ec9dd6fc76202c94d23942da3cf624325a178e1b0125e70db90b8d7f15" +checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "base64 0.21.7", + "base64 0.21.4", "bincode", - "bitflags 2.5.0", + "bitflags 2.4.0", "blake3", "borsh 0.10.3", "borsh 0.9.3", @@ -5893,7 +5783,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.2.14", + "getrandom 0.2.10", "itertools 0.10.5", "js-sys", "lazy_static", @@ -5901,7 +5791,7 @@ dependencies = [ "libsecp256k1", "light-poseidon", "log", - "memoffset 0.9.1", + "memoffset 0.9.0", "num-bigint 0.4.4", "num-derive 0.3.3", "num-traits", @@ -5926,11 +5816,11 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50a6da7b501117f68ef51fc113d771b52af646dc42c43af23a85e32461d59c9" +checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" dependencies = [ - "base64 0.21.7", + "base64 0.21.4", "bincode", "eager", "enum-iterator", @@ -5954,9 +5844,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf920143eb7d5bd5f2ea8fc6ab8a1f521512dfe50af7833af40d7cbae83d955d" +checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" dependencies = [ "crossbeam-channel", "futures-util", @@ -5979,9 +5869,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "627491c0afd615efb2538c8a49948663ac01aadf99a3cfebb0a63e2b9431ed79" +checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" dependencies = [ "async-mutex", "async-trait", @@ -5992,7 +5882,7 @@ dependencies = [ "quinn", "quinn-proto", "rcgen", - "rustls 0.21.10", + "rustls 0.21.7", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -6006,9 +5896,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5c306f32e26031c043c218a0ba3cde08cbb0e08511ab8a4128445d92a535e0" +checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" dependencies = [ "lazy_static", "num_cpus", @@ -6016,9 +5906,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c9dbb8cca1b917a01979a1ca98b682f6123df56f11a5227a37c36494ad7740" +checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" dependencies = [ "console", "dialoguer", @@ -6035,12 +5925,12 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2edc8c0f26561e770f28edb9d456221a5554ee90eeb9052ed38942dbb31c035b" +checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" dependencies = [ "async-trait", - "base64 0.21.7", + "base64 0.21.4", "bincode", "bs58 0.4.0", "indicatif", @@ -6061,11 +5951,11 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff63ab63805a6302ec21284a1138daf5478020f79641d77e53bb84f16c1f5a6" +checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" dependencies = [ - "base64 0.21.7", + "base64 0.21.4", "bs58 0.4.0", "jsonrpc-core", "reqwest", @@ -6083,9 +5973,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897db0c1085b8a99ca35871132b6dc6dca3eff68d9210fcd168ccc2e3843dc26" +checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -6096,14 +5986,14 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368430d6c9f033e86f8f590d19232d10986d1188c3ad3a6836628d2acc09c21a" +checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" dependencies = [ "assert_matches", - "base64 0.21.7", + "base64 0.21.4", "bincode", - "bitflags 2.5.0", + "bitflags 2.4.0", "borsh 0.10.3", "bs58 0.4.0", "bytemuck", @@ -6150,15 +6040,15 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f554d2a144bb0138cfdeced9961cc8a09aaa09f0c3c9a63bd10da41c4a06d420" +checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" dependencies = [ "bs58 0.4.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -6169,16 +6059,16 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28e8941bc0b137f851626ff4d4ce9ebcf9be10619b48d2ed83cd1953af21821" +checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" dependencies = [ "async-channel 1.9.0", "bytes", "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.2.6", + "indexmap 2.0.2", "itertools 0.10.5", "libc", "log", @@ -6190,7 +6080,7 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rcgen", - "rustls 0.21.10", + "rustls 0.21.7", "solana-metrics", "solana-perf", "solana-sdk", @@ -6201,9 +6091,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760b94e2875b54350f786c85faf50fa1a9a37ae46e9897215d19b373fc2f58cd" +checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" dependencies = [ "bincode", "log", @@ -6216,14 +6106,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cfdc7ff6cdc1512661de1f9f40723f88dc1e94c8b8938bd537f3713239173e2" +checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.2.6", + "indexmap 2.0.2", "indicatif", "log", "rayon", @@ -6240,12 +6130,12 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7131d11c8d5a068bfc26a9dc8c9ee0d77eaf60856dd0c8be880542fc5fbbd6" +checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" dependencies = [ "Inflector", - "base64 0.21.7", + "base64 0.21.4", "bincode", "borsh 0.10.3", "bs58 0.4.0", @@ -6265,9 +6155,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54647340d7fa1708c217cbc2411259c5b3784c2df55c1eb4353aca296635ed87" +checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" dependencies = [ "async-trait", "solana-connection-cache", @@ -6280,9 +6170,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de7a6bad7dff1c595be2eec0c3800a482c6068f3c87c6df87ed091b4e6e642e" +checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" dependencies = [ "log", "rustc_version", @@ -6296,9 +6186,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c828d118d1f135baacfbf51412c4f1ac18af19fdbee43b002d2908e69cdf50b" +checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" dependencies = [ "bincode", "log", @@ -6318,12 +6208,12 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112944743b08f7e1101368ff6d84745e7b4abb075fabaccc02e01bd3ce4b6d6c" +checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" dependencies = [ "aes-gcm-siv", - "base64 0.21.7", + "base64 0.21.4", "bincode", "bytemuck", "byteorder", @@ -6406,7 +6296,7 @@ checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f" dependencies = [ "assert_matches", "borsh 0.10.3", - "num-derive 0.4.2", + "num-derive 0.4.1", "num-traits", "solana-program", "spl-token", @@ -6427,9 +6317,9 @@ dependencies = [ [[package]] name = "spl-discriminator" -version = "0.1.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa600f2fe56f32e923261719bae640d873edadbc5237681a39b8e37bfd4d263" +checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f" dependencies = [ "bytemuck", "solana-program", @@ -6438,25 +6328,25 @@ dependencies = [ [[package]] name = "spl-discriminator-derive" -version = "0.1.2" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" +checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" dependencies = [ "quote", "spl-discriminator-syn", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "spl-discriminator-syn" -version = "0.1.2" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63" +checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.58", + "syn 2.0.38", "thiserror", ] @@ -6480,11 +6370,11 @@ dependencies = [ [[package]] name = "spl-pod" -version = "0.1.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5db7e4efb1107b0b8e52a13f035437cdcb36ef99c58f6d467f089d9b2915a" +checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" dependencies = [ - "base64 0.21.7", + "base64 0.21.4", "borsh 0.10.3", "bytemuck", "serde", @@ -6495,11 +6385,11 @@ dependencies = [ [[package]] name = "spl-program-error" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0657b6490196971d9e729520ba934911ff41fbb2cb9004463dbe23cf8b4b4f" +checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" dependencies = [ - "num-derive 0.4.2", + "num-derive 0.4.1", "num-traits", "solana-program", "spl-program-error-derive", @@ -6508,14 +6398,14 @@ dependencies = [ [[package]] name = "spl-program-error-derive" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474" +checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -6555,7 +6445,7 @@ checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059" dependencies = [ "arrayref", "bytemuck", - "num-derive 0.4.2", + "num-derive 0.4.1", "num-traits", "num_enum 0.7.2", "solana-program", @@ -6616,9 +6506,9 @@ dependencies = [ [[package]] name = "spl-type-length-value" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9ebd75d29c5f48de5f6a9c114e08531030b75b8ac2c557600ac7da0b73b1e8" +checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac" dependencies = [ "bytemuck", "solana-program", @@ -6629,11 +6519,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" dependencies = [ - "itertools 0.12.1", + "itertools 0.11.0", "nom", "unicode_categories", ] @@ -6654,7 +6544,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.8", + "ahash 0.7.6", "atoi", "base64 0.13.1", "bitflags 1.3.2", @@ -6698,7 +6588,7 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", - "time 0.3.35", + "time 0.3.29", "tokio-stream", "url", "uuid", @@ -6752,7 +6642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bcb902b974bc20b50c3ad3148022a366a46c9a676b587684ff46c237a3329e" dependencies = [ "async-channel 1.9.0", - "async-io 1.13.0", + "async-io", "atomic 0.5.3", "crossbeam-channel", "futures", @@ -6787,12 +6677,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subtle" version = "2.4.1" @@ -6812,33 +6696,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.58" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "syn_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.58", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "synstructure" version = "0.12.6" @@ -6880,21 +6746,22 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", - "fastrand 2.0.2", - "rustix 0.38.32", - "windows-sys 0.52.0", + "fastrand 2.0.1", + "redox_syscall 0.3.5", + "rustix 0.38.18", + "windows-sys 0.48.0", ] [[package]] name = "termcolor" -version = "1.4.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" dependencies = [ "winapi-util", ] @@ -6910,35 +6777,35 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ "cfg-if", "once_cell", @@ -6957,14 +6824,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.35" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef89ece63debf11bc32d1ed8d078ac870cbeb44da02afb02a9ff135ae7ca0582" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", - "num-conv", - "powerfmt", "serde", "time-core", "time-macros", @@ -6978,11 +6843,10 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ - "num-conv", "time-core", ] @@ -7022,9 +6886,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -7034,7 +6898,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.6", + "socket2 0.5.4", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -7042,13 +6906,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -7078,15 +6942,15 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.10", + "rustls 0.21.7", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -7101,18 +6965,18 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.10", + "rustls 0.21.7", "tokio", "tokio-rustls 0.24.1", "tungstenite", - "webpki-roots 0.25.4", + "webpki-roots 0.25.2", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -7134,14 +6998,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.9", + "toml_edit 0.21.0", ] [[package]] @@ -7159,33 +7023,22 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", + "indexmap 2.0.2", "toml_datetime", - "winnow 0.5.40", + "winnow", ] [[package]] name = "toml_edit" -version = "0.22.9" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.0.2", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.5", + "winnow", ] [[package]] @@ -7231,10 +7084,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -7243,20 +7097,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -7311,9 +7165,9 @@ dependencies = [ "anchor-client", "anyhow", "bs58 0.4.0", - "clap 4.5.4", + "clap 4.4.8", "digital_asset_types", - "env_logger 0.10.2", + "env_logger 0.10.0", "flatbuffers", "futures", "hex", @@ -7335,9 +7189,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.11" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" +checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" dependencies = [ "serde", "stable_deref_trait", @@ -7345,9 +7199,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" @@ -7362,7 +7216,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.10", + "rustls 0.21.7", "sha1", "thiserror", "url", @@ -7375,8 +7229,8 @@ name = "txn_forwarder" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.5.4", - "env_logger 0.10.2", + "clap 4.4.8", + "env_logger 0.10.0", "figment", "flatbuffers", "futures", @@ -7403,9 +7257,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uncased" -version = "0.9.10" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" dependencies = [ "version_check", ] @@ -7421,9 +7275,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" @@ -7433,18 +7287,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -7485,9 +7339,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.11" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" [[package]] name = "untrusted" @@ -7513,9 +7367,9 @@ dependencies = [ [[package]] name = "url" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -7536,11 +7390,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.8.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.10", "serde", ] @@ -7552,9 +7406,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.8.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74797339c3b98616c009c7c3eb53a0ce41e85c8ec66bd3db96ed132d20cfdee8" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" [[package]] name = "vcpkg" @@ -7595,12 +7449,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" - [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -7619,17 +7467,11 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7637,24 +7479,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -7664,9 +7506,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7674,28 +7516,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -7707,7 +7549,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.8", + "ring 0.17.2", "untrusted 0.9.0", ] @@ -7731,9 +7573,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "wg" @@ -7748,12 +7590,11 @@ dependencies = [ [[package]] name = "whoami" -version = "1.5.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" dependencies = [ - "redox_syscall 0.4.1", - "wasite", + "wasm-bindgen", "web-sys", ] @@ -7789,12 +7630,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", ] [[package]] @@ -7807,12 +7657,18 @@ dependencies = [ ] [[package]] -name = "windows-sys" -version = "0.52.0" +name = "windows-targets" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows-targets 0.52.4", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -7831,19 +7687,10 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.52.4" +name = "windows_aarch64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" -dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", -] +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -7852,10 +7699,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.4" +name = "windows_aarch64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -7864,10 +7711,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_aarch64_msvc" -version = "0.52.4" +name = "windows_i686_gnu" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -7876,10 +7723,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_gnu" -version = "0.52.4" +name = "windows_i686_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -7888,10 +7735,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_i686_msvc" -version = "0.52.4" +name = "windows_x86_64_gnu" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -7900,10 +7747,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnu" -version = "0.52.4" +name = "windows_x86_64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -7912,10 +7759,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.4" +name = "windows_x86_64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" @@ -7923,26 +7770,11 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" - [[package]] name = "winnow" -version = "0.5.40" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" dependencies = [ "memchr", ] @@ -7981,20 +7813,29 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.35", + "time 0.3.29", ] [[package]] name = "xxhash-rust" -version = "0.8.10" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" + +[[package]] +name = "yaml-rust" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] [[package]] name = "yansi" -version = "1.0.1" +version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" [[package]] name = "yasna" @@ -8002,7 +7843,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.35", + "time 0.3.29", ] [[package]] @@ -8022,7 +7863,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -8042,7 +7883,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.38", ] [[package]] @@ -8066,10 +7907,11 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", + "libc", "pkg-config", ] diff --git a/das_api/Cargo.toml b/das_api/Cargo.toml index d41028193..566ac1cd8 100644 --- a/das_api/Cargo.toml +++ b/das_api/Cargo.toml @@ -12,39 +12,25 @@ blockbuster = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } -digital_asset_types = { workspace = true, features = [ - "json_types", - "sql_types", -] } -env_logger = { workspace = true } +digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +env_logger = { workspace = true } figment = { workspace = true, features = ["env"] } hyper = { workspace = true } -jsonrpsee = { workspace = true, features = ["server", "macros"] } -jsonrpsee-core = { workspace = true, features = ["server"] } +jsonrpsee = { workspace = true, features = ["server", "macros"]} +jsonrpsee-core = { workspace = true, features =["server"]} log = { workspace = true } metrics = { workspace = true } mpl-bubblegum = { workspace = true } -mpl-token-metadata = { workspace = true, features = ["serde"] } +mpl-token-metadata = { workspace = true, features = ["serde"] } open-rpc-derive = { workspace = true } open-rpc-schema = { workspace = true } schemars = { workspace = true } schemars_derive = { workspace = true } -sea-orm = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "sqlx-postgres", -] } +sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres"] } serde = { workspace = true } serde_json = { workspace = true } solana-sdk = { workspace = true } -sqlx = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "postgres", - "uuid", - "offline", - "json", -] } +sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["signal"] } tower = { workspace = true } diff --git a/digital_asset_types/Cargo.toml b/digital_asset_types/Cargo.toml index 74fb32353..449f6e78d 100644 --- a/digital_asset_types/Cargo.toml +++ b/digital_asset_types/Cargo.toml @@ -8,7 +8,7 @@ publish = { workspace = true } [dependencies] async-trait = { workspace = true } blockbuster = { workspace = true } -borsh = { workspace = true, optional = true } +borsh = { workspace = true, optional = true } borsh-derive = { workspace = true, optional = true } bs58 = { workspace = true } futures = { workspace = true } @@ -20,18 +20,10 @@ num-derive = { workspace = true } num-traits = { workspace = true } schemars = { workspace = true } schemars_derive = { workspace = true } -sea-orm = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "sqlx-postgres", - "with-chrono", - "mock", -], optional = true } +sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"], optional = true } sea-query = { workspace = true, features = ["postgres-array"] } serde = { workspace = true, optional = true } -serde_json = { workspace = true, features = [ - "preserve_order", -], optional = true } +serde_json = { workspace = true, features = ["preserve_order"], optional = true } solana-sdk = { workspace = true } spl-concurrent-merkle-tree = { workspace = true } thiserror = { workspace = true } diff --git a/digital_asset_types/src/dao/scopes/asset.rs b/digital_asset_types/src/dao/scopes/asset.rs index 2df576591..58e4cbbaf 100644 --- a/digital_asset_types/src/dao/scopes/asset.rs +++ b/digital_asset_types/src/dao/scopes/asset.rs @@ -1,7 +1,10 @@ use crate::{ dao::{ - asset, asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, - extensions, Cursor, FullAsset, GroupingSize, Pagination, + asset::{self}, + asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, + extensions::{self, instruction::PascalCase}, + sea_orm_active_enums::Instruction, + Cursor, FullAsset, GroupingSize, Pagination, }, rpc::filter::AssetSortDirection, }; @@ -462,8 +465,7 @@ pub async fn fetch_transactions( .into_iter() .map(|transaction| { let tx = bs58::encode(transaction.tx).into_string(); - let ix = extensions::instruction::PascalCase::to_pascal_case(&transaction.instruction) - .to_string(); + let ix = Instruction::to_pascal_case(&transaction.instruction).to_string(); (tx, ix) }) .collect(); diff --git a/metaplex-rpc-proxy/Cargo.toml b/metaplex-rpc-proxy/Cargo.toml index 2e69785dd..06b801543 100644 --- a/metaplex-rpc-proxy/Cargo.toml +++ b/metaplex-rpc-proxy/Cargo.toml @@ -13,8 +13,6 @@ lazy_static = { workspace = true } log = { workspace = true } proxy-wasm = { workspace = true } regex = { workspace = true } -wasi = { workspace = true } -wasm-bindgen = { workspace = true } [lints] workspace = true diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index db41c0a1f..1eb403ca0 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -8,7 +8,6 @@ publish = { workspace = true } [dependencies] async-trait = { workspace = true } -blockbuster = { workspace = true } borsh = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs index fcf27fad3..7d7b32060 100644 --- a/nft_ingester/src/backfiller.rs +++ b/nft_ingester/src/backfiller.rs @@ -961,7 +961,7 @@ impl<'a, T: Messenger> Backfiller<'a, T> { // Filter out transactions that don't have to do with the tree we are interested in or // the Bubblegum program. let tb = tree.to_bytes(); - let bubblegum = blockbuster::programs::bubblegum::ID.to_bytes(); + let bubblegum = mpl_bubblegum::ID.to_bytes(); if account_keys.iter().all(|pk| *pk != tb && *pk != bubblegum) { continue; } diff --git a/tools/bgtask_creator/Cargo.toml b/tools/bgtask_creator/Cargo.toml index be662e34e..7011185e5 100644 --- a/tools/bgtask_creator/Cargo.toml +++ b/tools/bgtask_creator/Cargo.toml @@ -8,32 +8,16 @@ publish = { workspace = true } [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "cargo"] } -digital_asset_types = { workspace = true, features = [ - "json_types", - "sql_types", -] } +digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } futures = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } nft_ingester = { workspace = true } prometheus = { workspace = true } -sea-orm = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "sqlx-postgres", - "with-chrono", - "mock", -] } +sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] } sea-query = { workspace = true, features = ["postgres-array"] } solana-sdk = { workspace = true } -sqlx = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "postgres", - "uuid", - "offline", - "json", -] } +sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } txn_forwarder = { workspace = true } diff --git a/tools/load_generation/Cargo.toml b/tools/load_generation/Cargo.toml index 0c7a97d19..bd7b0de3b 100644 --- a/tools/load_generation/Cargo.toml +++ b/tools/load_generation/Cargo.toml @@ -12,9 +12,7 @@ rand = { workspace = true } solana-client = { workspace = true } solana-program = { workspace = true } solana-sdk = { workspace = true } -spl-associated-token-account = { workspace = true, features = [ - "no-entrypoint", -] } +spl-associated-token-account = { workspace = true, features = ["no-entrypoint"] } spl-token = { workspace = true, features = ["no-entrypoint"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/tools/tree-status/Cargo.toml b/tools/tree-status/Cargo.toml index 3ff2e0c37..da8ccc058 100644 --- a/tools/tree-status/Cargo.toml +++ b/tools/tree-status/Cargo.toml @@ -8,15 +8,11 @@ repository = { workspace = true } publish = { workspace = true } [dependencies] - anchor-client = { workspace = true } anyhow = { workspace = true } bs58 = { workspace = true } clap = { workspace = true, features = ["derive"] } -digital_asset_types = { workspace = true, features = [ - "json_types", - "sql_types", -] } +digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } env_logger = { workspace = true } flatbuffers = { workspace = true } futures = { workspace = true } @@ -24,35 +20,16 @@ hex = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } prometheus = { workspace = true } -sea-orm = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "sqlx-postgres", - "with-chrono", - "mock", -] } +sea-orm = { workspace = true, features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] } serde_json = { workspace = true } solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true, features = ["no-entrypoint"] } spl-noop = { workspace = true, features = ["no-entrypoint"] } +sqlx = { workspace = true, features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] } thiserror = { workspace = true } -tokio = { workspace = true, features = [ - "fs", - "macros", - "rt-multi-thread", - "sync", - "time", -] } -sqlx = { workspace = true, features = [ - "macros", - "runtime-tokio-rustls", - "postgres", - "uuid", - "offline", - "json", -] } +tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread", "sync", "time"] } txn_forwarder = { workspace = true } [lints] diff --git a/tools/txn_forwarder/Cargo.toml b/tools/txn_forwarder/Cargo.toml index 9a8388325..c9f5ca999 100644 --- a/tools/txn_forwarder/Cargo.toml +++ b/tools/txn_forwarder/Cargo.toml @@ -23,13 +23,7 @@ solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = [ - "macros", - "rt-multi-thread", - "sync", - "time", - "fs", -] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time", "fs"] } tokio-stream = { workspace = true, features = ["io-util"] } [lints] From 9d8031e86e2d6e8610284653c87d28c5fa164a06 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 23 Nov 2023 17:33:54 -0500 Subject: [PATCH 11/72] remove AccountInfo --- Cargo.lock | 1682 +++++++++++++++++--------------- nft_ingester/Cargo.toml | 2 +- nft_ingester/src/backfiller.rs | 13 +- nft_ingester/src/error/mod.rs | 50 +- 4 files changed, 970 insertions(+), 777 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a629c27a..730f88db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,8 +18,8 @@ version = "0.7.12" dependencies = [ "anyhow", "bs58 0.4.0", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.4", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -93,11 +93,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", "once_cell", "version_check", ] @@ -109,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.14", "once_cell", "version_check", "zerocopy", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f2a3e1df4685f18d12a943a9f2a7456305401af21a07c9fe076ef9ecd6e400" dependencies = [ "anchor-syn", - "bs58 0.5.0", + "bs58 0.5.1", "proc-macro2", "quote", "syn 1.0.109", @@ -295,7 +295,7 @@ dependencies = [ "bincode", "borsh 0.10.3", "bytemuck", - "getrandom 0.2.10", + "getrandom 0.2.14", "solana-program", "thiserror", ] @@ -307,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9101b84702fed2ea57bd22992f75065da5648017135b844283a2f6d74f27825" dependencies = [ "anyhow", - "bs58 0.5.0", + "bs58 0.5.1", "heck 0.3.3", "proc-macro2", "quote", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -358,49 +358,49 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "ark-bn254" @@ -550,7 +550,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.29", + "time 0.3.35", ] [[package]] @@ -605,22 +605,22 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener 3.1.0", - "event-listener-strategy", + "event-listener 5.3.0", + "event-listener-strategy 0.5.1", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" dependencies = [ "brotli", "flate2", @@ -632,30 +632,30 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.7.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" +checksum = "5f98c37cf288e302c16ef6c8472aad1e034c6c84ce5ea7b8101c98eb4a802fee" dependencies = [ - "async-lock 3.1.1", + "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.0.1", + "fastrand 2.0.2", + "futures-lite 2.3.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 1.9.0", + "async-channel 2.2.0", "async-executor", - "async-io", - "async-lock 2.8.0", + "async-io 2.3.2", + "async-lock 3.3.0", "blocking", - "futures-lite 1.13.0", + "futures-lite 2.3.0", "once_cell", "tokio", ] @@ -673,13 +673,32 @@ dependencies = [ "futures-lite 1.13.0", "log", "parking", - "polling", + "polling 2.8.0", "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.3.0", + "parking", + "polling 3.6.0", + "rustix 0.38.32", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" @@ -691,12 +710,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.1.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ - "event-listener 3.1.0", - "event-listener-strategy", + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", "pin-project-lite", ] @@ -718,7 +737,7 @@ dependencies = [ "async-attributes", "async-channel 1.9.0", "async-global-executor", - "async-io", + "async-io 1.13.0", "async-lock 2.8.0", "crossbeam-utils", "futures-channel", @@ -755,24 +774,24 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "async-task" -version = "4.5.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -818,17 +837,17 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "backon" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ - "fastrand 1.9.0", + "fastrand 2.0.2", "futures-core", "pin-project", "tokio", @@ -836,9 +855,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -876,9 +895,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -906,7 +925,7 @@ name = "bgtask_creator" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", "futures", "lazy_static", @@ -938,9 +957,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" dependencies = [ "serde", ] @@ -968,9 +987,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", @@ -1041,12 +1060,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.1.0", - "async-lock 3.1.1", + "async-channel 2.2.0", + "async-lock 3.3.0", "async-task", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", - "futures-lite 2.0.1", + "futures-lite 2.3.0", "piper", "tracing", ] @@ -1071,6 +1090,16 @@ dependencies = [ "hashbrown 0.13.2", ] +[[package]] +name = "borsh" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0901fc8eb0aca4c83be0106d6f2db17d86a08dfc2c25f0e84464bf381158add6" +dependencies = [ + "borsh-derive 1.4.0", + "cfg_aliases", +] + [[package]] name = "borsh-derive" version = "0.9.3" @@ -1097,6 +1126,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51670c3aa053938b0ee3bd67c3817e471e626151131b934038e83c5bf8de48f5" +dependencies = [ + "once_cell", + "proc-macro-crate 3.1.0", + "proc-macro2", + "quote", + "syn 2.0.58", + "syn_derive", +] + [[package]] name = "borsh-derive-internal" version = "0.9.3" @@ -1143,9 +1186,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1154,9 +1197,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" +checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1170,18 +1213,18 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", ] [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "serde", @@ -1189,9 +1232,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bv" @@ -1205,9 +1248,9 @@ dependencies = [ [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1216,9 +1259,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", @@ -1236,13 +1279,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1253,9 +1296,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cadence" @@ -1287,9 +1330,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" dependencies = [ "jobserver", "libc", @@ -1301,11 +1344,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1313,7 +1362,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.4", ] [[package]] @@ -1354,29 +1403,29 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.0", + "textwrap 0.16.1", ] [[package]] name = "clap" -version = "4.4.8" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", - "clap_derive 4.4.7", + "clap_derive 4.5.4", ] [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", - "strsim 0.10.0", + "clap_lex 0.7.0", + "strsim 0.11.1", ] [[package]] @@ -1394,14 +1443,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1415,9 +1464,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -1440,9 +1489,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", @@ -1454,24 +1503,24 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -1508,9 +1557,9 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1518,95 +1567,85 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1659,9 +1698,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ "darling_core", "darling_macro", @@ -1669,27 +1708,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1700,7 +1739,7 @@ dependencies = [ "backon", "cadence", "cadence-macros", - "clap 4.4.8", + "clap 4.5.4", "figment", "plerkle_messenger", "solana-account-decoder", @@ -1722,10 +1761,10 @@ dependencies = [ "borsh 0.10.3", "cadence", "cadence-macros", - "clap 4.4.8", + "clap 4.5.4", "das-core", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -1755,7 +1794,7 @@ dependencies = [ "cadence", "cadence-macros", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "figment", "hyper", "jsonrpsee", @@ -1797,17 +1836,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.9", ] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "der" @@ -1834,10 +1873,11 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -1953,7 +1993,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1976,7 +2016,7 @@ checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -1987,9 +2027,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "eager" @@ -2034,9 +2074,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" dependencies = [ "serde", ] @@ -2049,31 +2089,31 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -2091,9 +2131,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -2110,12 +2150,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2126,9 +2166,20 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "3.1.0" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" dependencies = [ "concurrent-queue", "parking", @@ -2137,11 +2188,21 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.3.0" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" +checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" dependencies = [ - "event-listener 3.1.0", + "event-listener 5.3.0", "pin-project-lite", ] @@ -2166,9 +2227,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "feature-probe" @@ -2183,7 +2244,7 @@ dependencies = [ "anyhow", "async-trait", "borsh 0.10.3", - "clap 4.4.8", + "clap 4.5.4", "mpl-bubblegum", "solana-account-decoder", "solana-client", @@ -2194,15 +2255,15 @@ dependencies = [ [[package]] name = "figment" -version = "0.10.11" +version = "0.10.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a014ac935975a70ad13a3bff2463b1c1b083b35ae4cb6309cfc59476aa7a181f" +checksum = "fdefe49ed1057d124dc81a0681c30dd07de56ad96e32adc7b64e8f28eaab31c4" dependencies = [ "atomic 0.6.0", "pear", "serde", "serde_yaml", - "toml 0.8.8", + "toml 0.8.12", "uncased", "version_check", ] @@ -2225,9 +2286,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -2256,9 +2317,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -2292,9 +2353,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2307,9 +2368,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2317,15 +2378,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2345,9 +2406,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2366,46 +2427,45 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-core", "futures-io", - "memchr", "parking", "pin-project-lite", ] [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2461,9 +2521,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -2474,21 +2534,21 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -2516,9 +2576,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -2526,7 +2586,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -2548,7 +2608,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2557,7 +2617,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2571,9 +2631,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash 0.8.5", "allocator-api2", @@ -2585,7 +2645,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -2623,9 +2683,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2641,9 +2701,9 @@ checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac 0.12.1", ] @@ -2680,9 +2740,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2691,9 +2751,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2726,9 +2786,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2741,7 +2801,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2750,14 +2810,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", ] @@ -2777,16 +2837,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -2806,9 +2866,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2843,25 +2903,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "serde", ] [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", "number_prefix", - "portable-atomic 1.4.3", + "portable-atomic 1.6.0", "unicode-width", ] @@ -2873,16 +2933,15 @@ checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "insta" -version = "1.35.1" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" +checksum = "3eab73f58e59ca6526037208f0e98851159ec1633cf17b6cd2e1f2c3fd5d53cc" dependencies = [ "console", "lazy_static", "linked-hash-map", "serde", "similar", - "yaml-rust", ] [[package]] @@ -2934,26 +2993,26 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.18", - "windows-sys 0.48.0", + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -2967,33 +3026,33 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -3114,18 +3173,18 @@ dependencies = [ [[package]] name = "kaigan" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a26f49495f94a283312e7ef45a243540ef20c9356bb01c8d84a61ac8ba5339b" +checksum = "e623cca1f0e2a0919032c1bdabbf81dd9aa34658d5066aca7bb90d608317ab91" dependencies = [ "borsh 0.10.3", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] @@ -3147,9 +3206,19 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] [[package]] name = "libsecp256k1" @@ -3225,9 +3294,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "load_generation" @@ -3246,9 +3315,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -3256,9 +3325,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" dependencies = [ "value-bag", ] @@ -3284,9 +3353,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memmap2" @@ -3308,9 +3377,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -3343,7 +3412,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "metrics-macros", "portable-atomic 0.3.20", ] @@ -3393,18 +3462,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3436,22 +3505,22 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", + "serde_with 3.7.0", "solana-program", "thiserror", ] [[package]] name = "mpl-token-metadata" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b2de608098eb2ef2a5392069dea83084967e25a4d69d0380a6bb02454fc0fe" +checksum = "caf0f61b553e424a6234af1268456972ee66c2222e1da89079242251fa7479e5" dependencies = [ "borsh 0.10.3", "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", + "serde_with 3.7.0", "solana-program", "thiserror", ] @@ -3479,12 +3548,13 @@ name = "nft_ingester" version = "0.7.2" dependencies = [ "async-trait", + "blockbuster", "borsh 0.10.3", "bs58 0.4.0", "cadence", "cadence-macros", "chrono", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", "figment", "flatbuffers", @@ -3592,6 +3662,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -3605,30 +3681,29 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -3649,9 +3724,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -3662,7 +3737,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", ] @@ -3693,7 +3768,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3702,10 +3777,10 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3716,9 +3791,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -3740,9 +3815,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open-rpc-derive" @@ -3770,11 +3845,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -3791,7 +3866,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3802,9 +3877,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -3871,7 +3946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.9", ] [[package]] @@ -3890,13 +3965,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -3927,9 +4002,9 @@ dependencies = [ [[package]] name = "pear" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a386cd715229d399604b50d1361683fe687066f42d56f54be995bc6868f71c" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", @@ -3938,14 +4013,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f0f13dac8069c139e8300a6510e3f4143ecf5259c60b116a9b271b4ca0d54" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -3959,9 +4034,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "percentage" @@ -3974,29 +4049,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -4011,7 +4086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", ] @@ -4028,9 +4103,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plain" @@ -4040,9 +4115,9 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plerkle_messenger" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f8e4e8975dcd2dbf94c7f84409096f0a0e5af287eabc9a212704e9e325ec84" +checksum = "a94c75e3178528c45bca0b7d515bdc6a3733b5f67d5b37b8a085cf6d7f81e3f2" dependencies = [ "async-mutex", "async-trait", @@ -4088,6 +4163,21 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "polling" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.3.9", + "pin-project-lite", + "rustix 0.38.32", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "polyval" version = "0.5.3" @@ -4106,14 +4196,20 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" dependencies = [ - "portable-atomic 1.4.3", + "portable-atomic 1.6.0", ] [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -4140,6 +4236,15 @@ dependencies = [ "toml_edit 0.19.15", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4166,9 +4271,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -4181,7 +4286,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "version_check", "yansi", ] @@ -4279,7 +4384,7 @@ checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -4293,7 +4398,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.10", "thiserror", "tokio", "tracing", @@ -4309,7 +4414,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.10", "rustls-native-certs", "slab", "thiserror", @@ -4325,16 +4430,16 @@ checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ "bytes", "libc", - "socket2 0.5.4", + "socket2 0.5.6", "tracing", "windows-sys 0.48.0", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4442,7 +4547,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", ] [[package]] @@ -4465,9 +4570,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -4475,9 +4580,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -4491,7 +4596,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.29", + "time 0.3.35", "yasna", ] @@ -4513,7 +4618,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes", - "combine 4.6.6", + "combine 4.6.7", "futures", "futures-util", "itoa", @@ -4539,34 +4644,34 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.14", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -4580,13 +4685,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.3", ] [[package]] @@ -4597,27 +4702,27 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "rend" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -4636,11 +4741,12 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.10", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -4651,7 +4757,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", "winreg", ] @@ -4672,26 +4778,28 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.2" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911b295d2d302948838c8ac142da1ee09fa7863163b44e6715bc9357905878b8" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "getrandom 0.2.10", + "cfg-if", + "getrandom 0.2.14", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -4703,9 +4811,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -4748,12 +4856,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.32.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" dependencies = [ "arrayvec", - "borsh 0.10.3", + "borsh 1.4.0", "bytes", "num-traits", "rand 0.8.5", @@ -4776,9 +4884,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-serialize" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" [[package]] name = "rustc_version" @@ -4814,15 +4922,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.10", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] @@ -4839,12 +4947,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -4863,49 +4971,49 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" dependencies = [ "dyn-clone", "schemars_derive", @@ -4915,9 +5023,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ "proc-macro2", "quote", @@ -4948,17 +5056,17 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -4983,7 +5091,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", - "time 0.3.29", + "time 0.3.35", "tracing", "url", "uuid", @@ -5044,7 +5152,7 @@ dependencies = [ "rust_decimal", "sea-query-derive 0.2.0", "serde_json", - "time 0.3.29", + "time 0.3.35", "uuid", ] @@ -5068,7 +5176,7 @@ dependencies = [ "sea-query 0.27.2", "serde_json", "sqlx", - "time 0.3.29", + "time 0.3.35", "uuid", ] @@ -5151,9 +5259,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -5164,9 +5272,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -5174,37 +5282,37 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -5220,11 +5328,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5232,9 +5340,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -5263,20 +5371,20 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.2", + "indexmap 2.2.6", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.6.1", - "time 0.3.29", + "serde_with_macros 3.7.0", + "time 0.3.35", ] [[package]] @@ -5288,28 +5396,28 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5338,7 +5446,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -5455,9 +5563,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "similar" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" [[package]] name = "sized-chunks" @@ -5480,15 +5588,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -5496,12 +5604,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5522,12 +5630,12 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" +checksum = "d145d4e1e33bfecd209059a0c4c75d623dbcaeb565b4c197f1815257be45726a" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "bv", @@ -5547,9 +5655,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" +checksum = "1deaf83f98be3ba9ecee057efa5cdfa6112267e5b1ff53c4ef4b727f66090b9a" dependencies = [ "chrono", "clap 2.34.0", @@ -5564,16 +5672,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" +checksum = "2a8912026a203ff0d90973e7363f141c6ce569484e06ee0a6f72992144263136" dependencies = [ "async-trait", "bincode", "dashmap 4.0.2", "futures", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "quinn", @@ -5597,9 +5705,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" +checksum = "4000f4717f86c5f9e1105378e3a6521db770d0ad68417f59960ca4b51103fcd0" dependencies = [ "bincode", "chrono", @@ -5611,15 +5719,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" +checksum = "f8b1a4d67c01e5c4464ed9bffb656dec2c88e9bfd7d45bf754f650150e5d81db" dependencies = [ "async-trait", "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "log", "rand 0.8.5", "rayon", @@ -5633,9 +5741,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" +checksum = "1790013c7969353000c22907fc21610adb3389a7c9a27a386ebe7fb32b2ad307" dependencies = [ "ahash 0.8.5", "blake3", @@ -5663,21 +5771,21 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" +checksum = "a3ed2b49a3dd03ddd5107d6e629e8e5895724227a057b3511bf0c107c6d48308" dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "solana-logger" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" +checksum = "bfc0d5b4f046d07e845b69178989a6b3bf168a82eeee006adb77391b339bce64" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5686,9 +5794,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" +checksum = "857178177c6b378bcfc35df6867a6eef211059f5e4ab01ee87355d6b7493b556" dependencies = [ "log", "solana-sdk", @@ -5696,9 +5804,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" +checksum = "1c68f5cbfbafd002b4d94728748f632a3bd27772ca5c7139710d65940c95477c" dependencies = [ "crossbeam-channel", "gethostname", @@ -5711,9 +5819,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" +checksum = "8ce93c50199f077df9d8fd4a82bbdfe654422e1d21e5eecb7f878c4e0078f92f" dependencies = [ "bincode", "clap 3.2.25", @@ -5723,7 +5831,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_derive", - "socket2 0.5.4", + "socket2 0.5.6", "solana-logger", "solana-sdk", "solana-version", @@ -5733,9 +5841,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" +checksum = "2a233bc2032f4b3f462fe13325cd1a336f355e1d9e793faada488db409cec0d6" dependencies = [ "ahash 0.8.5", "bincode", @@ -5762,17 +5870,17 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" +checksum = "4b2ae4ec9dd6fc76202c94d23942da3cf624325a178e1b0125e70db90b8d7f15" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.5.0", "blake3", "borsh 0.10.3", "borsh 0.9.3", @@ -5783,7 +5891,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.2.10", + "getrandom 0.2.14", "itertools 0.10.5", "js-sys", "lazy_static", @@ -5791,7 +5899,7 @@ dependencies = [ "libsecp256k1", "light-poseidon", "log", - "memoffset 0.9.0", + "memoffset 0.9.1", "num-bigint 0.4.4", "num-derive 0.3.3", "num-traits", @@ -5816,11 +5924,11 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" +checksum = "b50a6da7b501117f68ef51fc113d771b52af646dc42c43af23a85e32461d59c9" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bincode", "eager", "enum-iterator", @@ -5844,9 +5952,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" +checksum = "bf920143eb7d5bd5f2ea8fc6ab8a1f521512dfe50af7833af40d7cbae83d955d" dependencies = [ "crossbeam-channel", "futures-util", @@ -5869,9 +5977,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" +checksum = "627491c0afd615efb2538c8a49948663ac01aadf99a3cfebb0a63e2b9431ed79" dependencies = [ "async-mutex", "async-trait", @@ -5882,7 +5990,7 @@ dependencies = [ "quinn", "quinn-proto", "rcgen", - "rustls 0.21.7", + "rustls 0.21.10", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -5896,9 +6004,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" +checksum = "4d5c306f32e26031c043c218a0ba3cde08cbb0e08511ab8a4128445d92a535e0" dependencies = [ "lazy_static", "num_cpus", @@ -5906,9 +6014,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" +checksum = "f1c9dbb8cca1b917a01979a1ca98b682f6123df56f11a5227a37c36494ad7740" dependencies = [ "console", "dialoguer", @@ -5925,12 +6033,12 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" +checksum = "2edc8c0f26561e770f28edb9d456221a5554ee90eeb9052ed38942dbb31c035b" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "indicatif", @@ -5951,11 +6059,11 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" +checksum = "5ff63ab63805a6302ec21284a1138daf5478020f79641d77e53bb84f16c1f5a6" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", "reqwest", @@ -5973,9 +6081,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" +checksum = "897db0c1085b8a99ca35871132b6dc6dca3eff68d9210fcd168ccc2e3843dc26" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -5986,14 +6094,14 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" +checksum = "368430d6c9f033e86f8f590d19232d10986d1188c3ad3a6836628d2acc09c21a" dependencies = [ "assert_matches", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.5.0", "borsh 0.10.3", "bs58 0.4.0", "bytemuck", @@ -6040,15 +6148,15 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" +checksum = "f554d2a144bb0138cfdeced9961cc8a09aaa09f0c3c9a63bd10da41c4a06d420" dependencies = [ "bs58 0.4.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6059,16 +6167,16 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" +checksum = "e28e8941bc0b137f851626ff4d4ce9ebcf9be10619b48d2ed83cd1953af21821" dependencies = [ "async-channel 1.9.0", "bytes", "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.0.2", + "indexmap 2.2.6", "itertools 0.10.5", "libc", "log", @@ -6080,7 +6188,7 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rcgen", - "rustls 0.21.7", + "rustls 0.21.10", "solana-metrics", "solana-perf", "solana-sdk", @@ -6091,9 +6199,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" +checksum = "760b94e2875b54350f786c85faf50fa1a9a37ae46e9897215d19b373fc2f58cd" dependencies = [ "bincode", "log", @@ -6106,14 +6214,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" +checksum = "7cfdc7ff6cdc1512661de1f9f40723f88dc1e94c8b8938bd537f3713239173e2" dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "rayon", @@ -6130,12 +6238,12 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" +checksum = "ba7131d11c8d5a068bfc26a9dc8c9ee0d77eaf60856dd0c8be880542fc5fbbd6" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "borsh 0.10.3", "bs58 0.4.0", @@ -6155,9 +6263,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" +checksum = "54647340d7fa1708c217cbc2411259c5b3784c2df55c1eb4353aca296635ed87" dependencies = [ "async-trait", "solana-connection-cache", @@ -6170,9 +6278,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" +checksum = "1de7a6bad7dff1c595be2eec0c3800a482c6068f3c87c6df87ed091b4e6e642e" dependencies = [ "log", "rustc_version", @@ -6186,9 +6294,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" +checksum = "3c828d118d1f135baacfbf51412c4f1ac18af19fdbee43b002d2908e69cdf50b" dependencies = [ "bincode", "log", @@ -6208,12 +6316,12 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.28" +version = "1.17.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" +checksum = "112944743b08f7e1101368ff6d84745e7b4abb075fabaccc02e01bd3ce4b6d6c" dependencies = [ "aes-gcm-siv", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bytemuck", "byteorder", @@ -6296,7 +6404,7 @@ checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f" dependencies = [ "assert_matches", "borsh 0.10.3", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "solana-program", "spl-token", @@ -6317,9 +6425,9 @@ dependencies = [ [[package]] name = "spl-discriminator" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f" +checksum = "daa600f2fe56f32e923261719bae640d873edadbc5237681a39b8e37bfd4d263" dependencies = [ "bytemuck", "solana-program", @@ -6328,25 +6436,25 @@ dependencies = [ [[package]] name = "spl-discriminator-derive" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" +checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" dependencies = [ "quote", "spl-discriminator-syn", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "spl-discriminator-syn" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" +checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.38", + "syn 2.0.58", "thiserror", ] @@ -6370,11 +6478,11 @@ dependencies = [ [[package]] name = "spl-pod" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" +checksum = "85a5db7e4efb1107b0b8e52a13f035437cdcb36ef99c58f6d467f089d9b2915a" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "borsh 0.10.3", "bytemuck", "serde", @@ -6385,11 +6493,11 @@ dependencies = [ [[package]] name = "spl-program-error" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" +checksum = "7e0657b6490196971d9e729520ba934911ff41fbb2cb9004463dbe23cf8b4b4f" dependencies = [ - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "solana-program", "spl-program-error-derive", @@ -6398,14 +6506,14 @@ dependencies = [ [[package]] name = "spl-program-error-derive" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" +checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6445,7 +6553,7 @@ checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059" dependencies = [ "arrayref", "bytemuck", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "num_enum 0.7.2", "solana-program", @@ -6506,9 +6614,9 @@ dependencies = [ [[package]] name = "spl-type-length-value" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac" +checksum = "8f9ebd75d29c5f48de5f6a9c114e08531030b75b8ac2c557600ac7da0b73b1e8" dependencies = [ "bytemuck", "solana-program", @@ -6519,11 +6627,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.11.0", + "itertools 0.12.1", "nom", "unicode_categories", ] @@ -6544,7 +6652,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "atoi", "base64 0.13.1", "bitflags 1.3.2", @@ -6588,7 +6696,7 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", - "time 0.3.29", + "time 0.3.35", "tokio-stream", "url", "uuid", @@ -6642,7 +6750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bcb902b974bc20b50c3ad3148022a366a46c9a676b587684ff46c237a3329e" dependencies = [ "async-channel 1.9.0", - "async-io", + "async-io 1.13.0", "atomic 0.5.3", "crossbeam-channel", "futures", @@ -6677,6 +6785,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.4.1" @@ -6696,15 +6810,33 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -6746,22 +6878,21 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.18", - "windows-sys 0.48.0", + "fastrand 2.0.2", + "rustix 0.38.32", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -6777,35 +6908,35 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -6824,12 +6955,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "ef89ece63debf11bc32d1ed8d078ac870cbeb44da02afb02a9ff135ae7ca0582" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -6843,10 +6976,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -6886,9 +7020,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -6898,7 +7032,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.6", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -6906,13 +7040,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -6942,15 +7076,15 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.10", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -6965,18 +7099,18 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.7", + "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", "tungstenite", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", ] [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -6998,14 +7132,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit 0.22.9", ] [[package]] @@ -7023,22 +7157,33 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "toml_datetime", - "winnow", + "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +dependencies = [ + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.5", ] [[package]] @@ -7084,11 +7229,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -7097,20 +7241,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -7165,9 +7309,9 @@ dependencies = [ "anchor-client", "anyhow", "bs58 0.4.0", - "clap 4.4.8", + "clap 4.5.4", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "flatbuffers", "futures", "hex", @@ -7189,9 +7333,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" dependencies = [ "serde", "stable_deref_trait", @@ -7199,9 +7343,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -7216,7 +7360,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.7", + "rustls 0.21.10", "sha1", "thiserror", "url", @@ -7229,8 +7373,8 @@ name = "txn_forwarder" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.4", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -7257,9 +7401,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uncased" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" dependencies = [ "version_check", ] @@ -7275,9 +7419,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -7287,18 +7431,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" @@ -7339,9 +7483,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -7367,9 +7511,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -7390,11 +7534,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.14", "serde", ] @@ -7406,9 +7550,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.2" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "74797339c3b98616c009c7c3eb53a0ce41e85c8ec66bd3db96ed132d20cfdee8" [[package]] name = "vcpkg" @@ -7467,11 +7611,17 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7479,24 +7629,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -7506,9 +7656,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7516,28 +7666,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -7549,7 +7699,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.2", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7573,9 +7723,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "wg" @@ -7590,11 +7740,12 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.4.1", + "wasite", "web-sys", ] @@ -7630,21 +7781,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-targets 0.52.4", ] [[package]] @@ -7657,18 +7799,12 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.52.4", ] [[package]] @@ -7687,10 +7823,19 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-targets" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] [[package]] name = "windows_aarch64_gnullvm" @@ -7699,10 +7844,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -7711,10 +7856,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -7723,10 +7868,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -7735,10 +7880,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -7747,10 +7892,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -7759,10 +7904,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -7770,11 +7915,26 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" -version = "0.5.16" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" dependencies = [ "memchr", ] @@ -7813,29 +7973,20 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.29", + "time 0.3.35", ] [[package]] name = "xxhash-rust" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" - -[[package]] -name = "yaml-rust" -version = "0.4.5" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] +checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" [[package]] name = "yansi" -version = "1.0.0-rc.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yasna" @@ -7843,7 +7994,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.29", + "time 0.3.35", ] [[package]] @@ -7863,7 +8014,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -7883,7 +8034,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.58", ] [[package]] @@ -7907,11 +8058,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index 1eb403ca0..fd17a2bed 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -6,8 +6,8 @@ repository = { workspace = true } publish = { workspace = true } [dependencies] - async-trait = { workspace = true } +blockbuster = { workspace = true } borsh = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs index 7d7b32060..fd7d3e19c 100644 --- a/nft_ingester/src/backfiller.rs +++ b/nft_ingester/src/backfiller.rs @@ -628,9 +628,14 @@ impl<'a, T: Messenger> Backfiller<'a, T> { &mut self, btree: &BackfillTree, ) -> Result, IngesterError> { - let address = Pubkey::try_from(btree.unique_tree.tree.as_slice()).map_err(|error| { - IngesterError::DeserializationError(format!("failed to parse pubkey: {error:?}")) - })?; + let address = match Pubkey::try_from(btree.unique_tree.tree.as_slice()) { + Ok(pubkey) => pubkey, + Err(error) => { + return Err(IngesterError::DeserializationError(format!( + "failed to parse pubkey: {error:?}" + ))) + } + }; let slots = self.find_slots_via_address(&address).await?; let address = btree.unique_tree.tree.clone(); @@ -961,7 +966,7 @@ impl<'a, T: Messenger> Backfiller<'a, T> { // Filter out transactions that don't have to do with the tree we are interested in or // the Bubblegum program. let tb = tree.to_bytes(); - let bubblegum = mpl_bubblegum::ID.to_bytes(); + let bubblegum = blockbuster::programs::bubblegum::ID.to_bytes(); if account_keys.iter().all(|pk| *pk != tb && *pk != bubblegum) { continue; } diff --git a/nft_ingester/src/error/mod.rs b/nft_ingester/src/error/mod.rs index ddeae0ba6..37ed5f24b 100644 --- a/nft_ingester/src/error/mod.rs +++ b/nft_ingester/src/error/mod.rs @@ -1,15 +1,27 @@ -use { - crate::tasks::TaskData, plerkle_messenger::MessengerError, - plerkle_serialization::error::PlerkleSerializationError, sea_orm::DbErr, - tokio::sync::mpsc::error::SendError, -}; +use crate::tasks::TaskData; +use blockbuster::error::BlockbusterError; +use plerkle_messenger::MessengerError; +use plerkle_serialization::error::PlerkleSerializationError; +use sea_orm::{DbErr, TransactionError}; +use thiserror::Error; +use tokio::sync::mpsc::error::SendError; -#[derive(Debug, thiserror::Error)] +#[derive(Error, Debug, PartialEq, Eq)] pub enum IngesterError { + #[error("ChangeLog Event Malformed")] + ChangeLogEventMalformed, + #[error("Compressed Asset Event Malformed")] + CompressedAssetEventMalformed, #[error("Network Error: {0}")] BatchInitNetworkingError(String), + #[error("Error writing batch files")] + BatchInitIOError, + #[error("Storage listener error: ({msg})")] + StorageListenerError { msg: String }, #[error("Storage Write Error: {0}")] StorageWriteError(String), + #[error("NotImplemented")] + NotImplemented, #[error("Deserialization Error: {0}")] DeserializationError(String), #[error("Task Manager Error: {0}")] @@ -24,6 +36,12 @@ pub enum IngesterError { SerializatonError(String), #[error("Messenger error; {0}")] MessengerError(String), + #[error("Blockbuster Parsing error: {0}")] + ParsingError(String), + #[error("Database Error: {0}")] + DatabaseError(String), + #[error("Unknown Task Type: {0}")] + UnknownTaskType(String), #[error("BG Task Manager Not Started")] TaskManagerNotStarted, #[error("Unrecoverable task error: {0}")] @@ -32,6 +50,8 @@ pub enum IngesterError { CacheStorageWriteError(String), #[error("HttpError {status_code}")] HttpError { status_code: String }, + #[error("AssetIndex Error {0}")] + AssetIndexError(String), } impl From for IngesterError { @@ -52,12 +72,30 @@ impl From for IngesterError { } } +impl From for IngesterError { + fn from(err: BlockbusterError) -> Self { + IngesterError::ParsingError(err.to_string()) + } +} + +impl From for IngesterError { + fn from(_err: std::io::Error) -> Self { + IngesterError::BatchInitIOError + } +} + impl From for IngesterError { fn from(e: DbErr) -> Self { IngesterError::StorageWriteError(e.to_string()) } } +impl From> for IngesterError { + fn from(e: TransactionError) -> Self { + IngesterError::StorageWriteError(e.to_string()) + } +} + impl From> for IngesterError { fn from(err: SendError) -> Self { IngesterError::TaskManagerError(format!("Could not create task: {:?}", err.to_string())) From 096616d3355405b922198e6aa624fb842f4ed226 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 23 Nov 2023 22:11:52 -0500 Subject: [PATCH 12/72] remove plerkle from program_transformers --- nft_ingester/src/transaction_notifications.rs | 2 +- program_transformers/Cargo.toml | 5 ++++- program_transformers/src/bubblegum/decompress.rs | 16 ++++------------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/nft_ingester/src/transaction_notifications.rs b/nft_ingester/src/transaction_notifications.rs index df005bc56..5335de6dc 100644 --- a/nft_ingester/src/transaction_notifications.rs +++ b/nft_ingester/src/transaction_notifications.rs @@ -10,7 +10,7 @@ use { log::{debug, error}, plerkle_messenger::{ConsumptionType, Messenger, MessengerConfig, RecvData}, plerkle_serialization::root_as_transaction_info, - program_transformers::ProgramTransformer, + program_transformers::{error::ProgramTransformerResult, ProgramTransformer, TransactionInfo}, sqlx::{Pool, Postgres}, std::sync::Arc, tokio::{ diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index 35bab7a19..08f784632 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -10,7 +10,10 @@ blockbuster = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } futures = { workspace = true } heck = { workspace = true } mpl-bubblegum = { workspace = true } diff --git a/program_transformers/src/bubblegum/decompress.rs b/program_transformers/src/bubblegum/decompress.rs index 9db67e8b8..208e9aabc 100644 --- a/program_transformers/src/bubblegum/decompress.rs +++ b/program_transformers/src/bubblegum/decompress.rs @@ -17,18 +17,10 @@ pub async fn decompress<'c, T>( where T: ConnectionTrait + TransactionTrait, { - let id_bytes = bundle.keys.get(3).unwrap().0.as_slice(); + let id_bytes = bundle.keys.get(3).unwrap().to_bytes().to_vec(); // Partial update of asset table with just leaf. - upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.to_vec()).await?; - upsert_asset_with_compression_info( - txn, - id_bytes.to_vec(), - false, - false, - 1, - Some(id_bytes.to_vec()), - true, - ) - .await + upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.clone()).await?; + upsert_asset_with_compression_info(txn, id_bytes.clone(), false, false, 1, Some(id_bytes), true) + .await } From ef0b0e9c10b6372dc810286b9b4277233f65916b Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 26 Nov 2023 19:14:56 -0500 Subject: [PATCH 13/72] nft_ingester2: grpc2redis --- Cargo.lock | 765 +++++++++++++++++++++++++--- Cargo.toml | 20 +- nft_ingester2/Cargo.toml | 45 ++ nft_ingester2/build.rs | 38 ++ nft_ingester2/config-grpc2redis.yml | 24 + nft_ingester2/src/config.rs | 153 ++++++ nft_ingester2/src/grpc.rs | 147 ++++++ nft_ingester2/src/ingester.rs | 5 + nft_ingester2/src/main.rs | 75 +++ nft_ingester2/src/prom.rs | 106 ++++ nft_ingester2/src/redis.rs | 25 + nft_ingester2/src/tracing.rs | 32 ++ nft_ingester2/src/util.rs | 16 + nft_ingester2/src/version.rs | 22 + 14 files changed, 1408 insertions(+), 65 deletions(-) create mode 100644 nft_ingester2/Cargo.toml create mode 100644 nft_ingester2/build.rs create mode 100644 nft_ingester2/config-grpc2redis.yml create mode 100644 nft_ingester2/src/config.rs create mode 100644 nft_ingester2/src/grpc.rs create mode 100644 nft_ingester2/src/ingester.rs create mode 100644 nft_ingester2/src/main.rs create mode 100644 nft_ingester2/src/prom.rs create mode 100644 nft_ingester2/src/redis.rs create mode 100644 nft_ingester2/src/tracing.rs create mode 100644 nft_ingester2/src/util.rs create mode 100644 nft_ingester2/src/version.rs diff --git a/Cargo.lock b/Cargo.lock index 730f88db4..debbc5287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,6 +841,60 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "0.4.4" @@ -1328,6 +1382,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cargo-lock" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72" +dependencies = [ + "semver", + "serde", + "toml 0.7.8", + "url", +] + [[package]] name = "cc" version = "1.0.92" @@ -2274,6 +2340,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flatbuffers" version = "23.5.26" @@ -2538,6 +2610,32 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.14" @@ -2822,6 +2920,18 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2953,6 +3063,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "integration_tests" version = "0.1.0" @@ -3057,6 +3173,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonpath_lib" version = "0.3.0" @@ -3341,6 +3468,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -3525,6 +3658,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "native-tls" version = "0.2.11" @@ -3583,6 +3722,37 @@ dependencies = [ "url", ] +[[package]] +name = "nft_ingester2" +version = "0.7.2" +dependencies = [ + "anyhow", + "atty", + "cargo-lock", + "clap 4.5.4", + "futures", + "git-version", + "hyper", + "json5", + "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", + "opentelemetry_sdk", + "prometheus", + "redis 0.25.3", + "serde", + "serde_yaml", + "solana-sdk", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "vergen", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", + "yellowstone-grpc-tools", +] + [[package]] name = "nix" version = "0.26.4" @@ -3783,6 +3953,15 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -3887,6 +4066,87 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.2.6", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +dependencies = [ + "async-trait", + "futures-core", + "futures-util", + "opentelemetry", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float 4.2.0", + "percent-encoding", + "rand 0.8.5", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +dependencies = [ + "num-traits", +] + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -4047,6 +4307,61 @@ dependencies = [ "num", ] +[[package]] +name = "pest" +version = "2.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "pest_meta" +version = "2.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.8", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.2.6", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -4127,7 +4442,7 @@ dependencies = [ "figment", "futures", "log", - "redis", + "redis 0.22.3", "serde", "thiserror", ] @@ -4217,6 +4532,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +dependencies = [ + "proc-macro2", + "syn 2.0.58", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -4331,12 +4656,74 @@ dependencies = [ "thiserror", ] +[[package]] +name = "prost" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.58", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "prost-types" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +dependencies = [ + "prost", +] + [[package]] name = "protobuf" version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + [[package]] name = "proxy-wasm" version = "0.2.1" @@ -4633,6 +5020,29 @@ dependencies = [ "url", ] +[[package]] +name = "redis" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6472825949c09872e8f2c50bde59fcefc17748b6be5c90fd67cd8b4daca73bfd" +dependencies = [ + "async-trait", + "bytes", + "combine 4.6.7", + "futures-util", + "itoa", + "native-tls", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.5.6", + "tokio", + "tokio-native-tls", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -5285,6 +5695,9 @@ name = "semver" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -5630,9 +6043,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d145d4e1e33bfecd209059a0c4c75d623dbcaeb565b4c197f1815257be45726a" +checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" dependencies = [ "Inflector", "base64 0.21.7", @@ -5655,9 +6068,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1deaf83f98be3ba9ecee057efa5cdfa6112267e5b1ff53c4ef4b727f66090b9a" +checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" dependencies = [ "chrono", "clap 2.34.0", @@ -5672,9 +6085,9 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8912026a203ff0d90973e7363f141c6ce569484e06ee0a6f72992144263136" +checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" dependencies = [ "async-trait", "bincode", @@ -5705,9 +6118,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4000f4717f86c5f9e1105378e3a6521db770d0ad68417f59960ca4b51103fcd0" +checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" dependencies = [ "bincode", "chrono", @@ -5719,9 +6132,9 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b1a4d67c01e5c4464ed9bffb656dec2c88e9bfd7d45bf754f650150e5d81db" +checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" dependencies = [ "async-trait", "bincode", @@ -5741,9 +6154,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1790013c7969353000c22907fc21610adb3389a7c9a27a386ebe7fb32b2ad307" +checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" dependencies = [ "ahash 0.8.5", "blake3", @@ -5771,9 +6184,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ed2b49a3dd03ddd5107d6e629e8e5895724227a057b3511bf0c107c6d48308" +checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" dependencies = [ "proc-macro2", "quote", @@ -5783,9 +6196,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc0d5b4f046d07e845b69178989a6b3bf168a82eeee006adb77391b339bce64" +checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5794,9 +6207,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857178177c6b378bcfc35df6867a6eef211059f5e4ab01ee87355d6b7493b556" +checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" dependencies = [ "log", "solana-sdk", @@ -5804,9 +6217,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c68f5cbfbafd002b4d94728748f632a3bd27772ca5c7139710d65940c95477c" +checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" dependencies = [ "crossbeam-channel", "gethostname", @@ -5819,9 +6232,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce93c50199f077df9d8fd4a82bbdfe654422e1d21e5eecb7f878c4e0078f92f" +checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" dependencies = [ "bincode", "clap 3.2.25", @@ -5841,9 +6254,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a233bc2032f4b3f462fe13325cd1a336f355e1d9e793faada488db409cec0d6" +checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" dependencies = [ "ahash 0.8.5", "bincode", @@ -5870,9 +6283,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2ae4ec9dd6fc76202c94d23942da3cf624325a178e1b0125e70db90b8d7f15" +checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" dependencies = [ "ark-bn254", "ark-ec", @@ -5924,9 +6337,9 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50a6da7b501117f68ef51fc113d771b52af646dc42c43af23a85e32461d59c9" +checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" dependencies = [ "base64 0.21.7", "bincode", @@ -5952,9 +6365,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf920143eb7d5bd5f2ea8fc6ab8a1f521512dfe50af7833af40d7cbae83d955d" +checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" dependencies = [ "crossbeam-channel", "futures-util", @@ -5977,9 +6390,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "627491c0afd615efb2538c8a49948663ac01aadf99a3cfebb0a63e2b9431ed79" +checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" dependencies = [ "async-mutex", "async-trait", @@ -6004,9 +6417,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5c306f32e26031c043c218a0ba3cde08cbb0e08511ab8a4128445d92a535e0" +checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" dependencies = [ "lazy_static", "num_cpus", @@ -6014,9 +6427,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c9dbb8cca1b917a01979a1ca98b682f6123df56f11a5227a37c36494ad7740" +checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" dependencies = [ "console", "dialoguer", @@ -6033,9 +6446,9 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2edc8c0f26561e770f28edb9d456221a5554ee90eeb9052ed38942dbb31c035b" +checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" dependencies = [ "async-trait", "base64 0.21.7", @@ -6059,9 +6472,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff63ab63805a6302ec21284a1138daf5478020f79641d77e53bb84f16c1f5a6" +checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" dependencies = [ "base64 0.21.7", "bs58 0.4.0", @@ -6081,9 +6494,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897db0c1085b8a99ca35871132b6dc6dca3eff68d9210fcd168ccc2e3843dc26" +checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -6094,9 +6507,9 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368430d6c9f033e86f8f590d19232d10986d1188c3ad3a6836628d2acc09c21a" +checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6148,9 +6561,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f554d2a144bb0138cfdeced9961cc8a09aaa09f0c3c9a63bd10da41c4a06d420" +checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" dependencies = [ "bs58 0.4.0", "proc-macro2", @@ -6167,9 +6580,9 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28e8941bc0b137f851626ff4d4ce9ebcf9be10619b48d2ed83cd1953af21821" +checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" dependencies = [ "async-channel 1.9.0", "bytes", @@ -6199,9 +6612,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760b94e2875b54350f786c85faf50fa1a9a37ae46e9897215d19b373fc2f58cd" +checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" dependencies = [ "bincode", "log", @@ -6214,9 +6627,9 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cfdc7ff6cdc1512661de1f9f40723f88dc1e94c8b8938bd537f3713239173e2" +checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" dependencies = [ "async-trait", "bincode", @@ -6238,9 +6651,9 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7131d11c8d5a068bfc26a9dc8c9ee0d77eaf60856dd0c8be880542fc5fbbd6" +checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" dependencies = [ "Inflector", "base64 0.21.7", @@ -6263,9 +6676,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54647340d7fa1708c217cbc2411259c5b3784c2df55c1eb4353aca296635ed87" +checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" dependencies = [ "async-trait", "solana-connection-cache", @@ -6278,9 +6691,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de7a6bad7dff1c595be2eec0c3800a482c6068f3c87c6df87ed091b4e6e642e" +checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" dependencies = [ "log", "rustc_version", @@ -6294,9 +6707,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c828d118d1f135baacfbf51412c4f1ac18af19fdbee43b002d2908e69cdf50b" +checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" dependencies = [ "bincode", "log", @@ -6316,9 +6729,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.30" +version = "1.17.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112944743b08f7e1101368ff6d84745e7b4abb075fabaccc02e01bd3ce4b6d6c" +checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" dependencies = [ "aes-gcm-siv", "base64 0.21.7", @@ -6942,6 +7355,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 2.10.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.45" @@ -6961,7 +7396,9 @@ checksum = "ef89ece63debf11bc32d1ed8d078ac870cbeb44da02afb02a9ff135ae7ca0582" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -7038,6 +7475,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -7130,6 +7577,18 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + [[package]] name = "toml" version = "0.8.12" @@ -7158,6 +7617,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.2.6", + "serde", + "serde_spanned", "toml_datetime", "winnow 0.5.40", ] @@ -7186,12 +7647,79 @@ dependencies = [ "winnow 0.6.5", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls 0.21.10", + "rustls-native-certs", + "rustls-pemfile", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "tonic-health" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" +dependencies = [ + "async-stream", + "prost", + "tokio", + "tokio-stream", + "tonic", +] + [[package]] name = "tower" version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -7271,6 +7799,24 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -7399,6 +7945,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "uncased" version = "0.9.10" @@ -7520,6 +8072,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -7566,6 +8124,18 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "vergen" +version = "8.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525" +dependencies = [ + "anyhow", + "rustc_version", + "rustversion", + "time 0.3.35", +] + [[package]] name = "version_check" version = "0.9.4" @@ -7693,6 +8263,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.22.4" @@ -7997,6 +8577,65 @@ dependencies = [ "time 0.3.35", ] +[[package]] +name = "yellowstone-grpc-client" +version = "1.15.0+solana.1.17.28" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +dependencies = [ + "bytes", + "futures", + "thiserror", + "tonic", + "tonic-health", + "yellowstone-grpc-proto", +] + +[[package]] +name = "yellowstone-grpc-proto" +version = "1.14.0+solana.1.17.28" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +dependencies = [ + "anyhow", + "bincode", + "prost", + "protobuf-src", + "solana-account-decoder", + "solana-sdk", + "solana-transaction-status", + "tonic", + "tonic-build", +] + +[[package]] +name = "yellowstone-grpc-tools" +version = "1.0.0-rc.11+solana.1.17.28" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +dependencies = [ + "anyhow", + "async-trait", + "atty", + "cargo-lock", + "clap 4.5.4", + "futures", + "git-version", + "hyper", + "json5", + "lazy_static", + "prometheus", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "tokio-stream", + "tonic", + "tonic-health", + "tracing", + "tracing-subscriber", + "vergen", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", +] + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index 769cf02a4..3cf0fcbfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "migration", "nft_ingester", "ops", + "nft_ingester2", "program_transformers", "tools/acc_forwarder", "tools/bgtask_creator", @@ -25,18 +26,22 @@ repository = "https://github.com/metaplex-foundation/digital-asset-rpc-infrastru version = "0.7.2" [workspace.dependencies] + +backon = "0.4.1" +heck = "0.5.0" anchor-client = "0.29.0" anchor-lang = "0.29.0" anyhow = "1.0.75" async-std = "1.0.0" async-trait = "0.1.60" -backon = "0.4.1" blockbuster = "2.2.1" +atty = "0.2.14" borsh = "~0.10.3" borsh-derive = "~0.10.3" bs58 = "0.4.0" cadence = "0.29.0" cadence-macros = "0.29.0" +cargo-lock = "9.0.0" chrono = "0.4.19" clap = "4.2.2" das_api = { path = "das_api" } @@ -50,13 +55,14 @@ figment = "0.10.8" flatbuffers = "23.1.21" function_name = "0.3.0" futures = "0.3.28" -heck = "0.5.0" hex = "0.4.3" hyper = "0.14.23" indexmap = "1.9.3" indicatif = "0.17.5" insta = "1.34.0" itertools = "0.10.1" +git-version = "0.3.5" +json5 = "0.4.1" jsonpath_lib = "0.3.0" jsonrpsee = "0.16.2" jsonrpsee-core = "0.16.2" @@ -73,12 +79,16 @@ num-traits = "0.2.15" once_cell = "1.19.0" open-rpc-derive = "0.0.4" open-rpc-schema = "0.0.4" +opentelemetry = "0.21.0" +opentelemetry-jaeger = "0.20.0" +opentelemetry_sdk = "0.21.1" plerkle_messenger = "1.6.0" plerkle_serialization = "1.8.0" program_transformers = { path = "program_transformers" } prometheus = "0.13.3" proxy-wasm = "0.2.0" rand = "0.8.5" +redis = "0.25.3" regex = "1.6.0" reqwest = "0.11.13" rust-crypto = "0.2.36" @@ -89,6 +99,7 @@ sea-orm-migration = "0.10.6" sea-query = "0.28.1" serde = "1.0.137" serde_json = "1.0.81" +serde_yaml = "0.9.34" serial_test = "2.0.0" solana-account-decoder = "~1.17" solana-client = "~1.17" @@ -108,11 +119,16 @@ tokio-stream = "0.1.14" tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" +tracing-opentelemetry = "0.22.0" tracing-subscriber = "0.3.16" txn_forwarder = { path = "tools/txn_forwarder" } url = "2.3.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" +vergen = "8.2.1" +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin +yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28", default-features = false } # tag is geyser plugin [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml new file mode 100644 index 000000000..5a4e0f142 --- /dev/null +++ b/nft_ingester2/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "nft_ingester2" +version = { workspace = true } +edition = { workspace = true } +repository = { workspace = true } +publish = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +atty = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive"] } +futures = { workspace = true } +hyper = { workspace = true, features = ["server"] } +json5 = { workspace = true } +lazy_static = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-jaeger = { workspace = true, features = ["rt-tokio"] } +opentelemetry_sdk = { workspace = true, features = ["trace"] } +prometheus = { workspace = true } +redis = { workspace = true, features = ["tokio-comp", "tokio-native-tls-comp"] } +serde = { workspace = true } +serde_yaml = { workspace = true } +solana-sdk = { workspace = true } # only prom rn +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "time", + "fs", + "tracing", +] } +tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } +yellowstone-grpc-client = { workspace = true } +yellowstone-grpc-proto = { workspace = true } +yellowstone-grpc-tools = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +cargo-lock = { workspace = true } +git-version = { workspace = true } +vergen = { workspace = true, features = ["build", "rustc"] } + +[lints] +workspace = true diff --git a/nft_ingester2/build.rs b/nft_ingester2/build.rs new file mode 100644 index 000000000..92e1f4c7c --- /dev/null +++ b/nft_ingester2/build.rs @@ -0,0 +1,38 @@ +use {cargo_lock::Lockfile, std::collections::HashSet}; + +fn main() -> anyhow::Result<()> { + let mut envs = vergen::EmitBuilder::builder(); + envs.all_build().all_rustc(); + envs.emit()?; + + // vergen git version does not looks cool + println!( + "cargo:rustc-env=GIT_VERSION={}", + git_version::git_version!() + ); + + // Extract packages version + let lockfile = Lockfile::load("../Cargo.lock")?; + println!( + "cargo:rustc-env=SOLANA_SDK_VERSION={}", + get_pkg_version(&lockfile, "solana-sdk") + ); + println!( + "cargo:rustc-env=YELLOWSTONE_GRPC_PROTO_VERSION={}", + get_pkg_version(&lockfile, "yellowstone-grpc-proto") + ); + + Ok(()) +} + +fn get_pkg_version(lockfile: &Lockfile, pkg_name: &str) -> String { + lockfile + .packages + .iter() + .filter(|pkg| pkg.name.as_str() == pkg_name) + .map(|pkg| pkg.version.to_string()) + .collect::>() + .into_iter() + .collect::>() + .join(",") +} diff --git a/nft_ingester2/config-grpc2redis.yml b/nft_ingester2/config-grpc2redis.yml new file mode 100644 index 000000000..73ed908f4 --- /dev/null +++ b/nft_ingester2/config-grpc2redis.yml @@ -0,0 +1,24 @@ +prometheus: 127.0.0.1:8873 +endpoint: http://127.0.0.1:10000 +x_token: null +commitment: processed +accounts: + stream: ACCOUNTS + stream_maxlen: 100_000_000 + stream_data_key: data + filters: + - owner: + - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA + - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s +transactions: + stream: TRANSACTIONS + stream_maxlen: 10_000_000 + stream_data_key: data + filters: + - account_include: + - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY +redis: + url: redis://localhost:6379 + pipeline_max_size: 10 + pipeline_max_idle_ms: 10 + max_xadd_in_process: 100 diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs new file mode 100644 index 000000000..14b45f247 --- /dev/null +++ b/nft_ingester2/src/config.rs @@ -0,0 +1,153 @@ +use { + anyhow::Context, + serde::{de, Deserialize}, + std::{net::SocketAddr, path::Path, time::Duration}, + tokio::fs, + yellowstone_grpc_tools::config::{ + deserialize_usize_str, ConfigGrpcRequestAccounts, ConfigGrpcRequestCommitment, + ConfigGrpcRequestTransactions, + }, +}; + +pub const REDIS_STREAM_ACCOUNTS: &str = "ACCOUNTS"; +pub const REDIS_STREAM_TRANSACTIONS: &str = "TRANSACTIONS"; + +pub async fn load(path: impl AsRef + Copy) -> anyhow::Result +where + T: de::DeserializeOwned, +{ + let text = fs::read_to_string(path) + .await + .context("failed to read config from file")?; + + match path.as_ref().extension().and_then(|e| e.to_str()) { + Some("yaml") | Some("yml") => { + serde_yaml::from_str(&text).context("failed to parse config from YAML file") + } + Some("json") => json5::from_str(&text).context("failed to parse config from JSON file"), + value => anyhow::bail!("unknown config extension: {value:?}"), + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +pub struct ConfigPrometheus { + pub prometheus: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ConfigGrpc { + pub endpoint: String, + pub x_token: Option, + + pub commitment: ConfigGrpcRequestCommitment, + pub accounts: ConfigGrpcAccounts, + pub transactions: ConfigGrpcTransactions, + + pub redis: ConfigGrpcRedis, +} + +#[derive(Debug, Deserialize)] +pub struct ConfigGrpcAccounts { + #[serde(default = "ConfigGrpcAccounts::default_stream")] + pub stream: String, + #[serde( + default = "ConfigGrpcAccounts::default_stream_maxlen", + deserialize_with = "deserialize_usize_str" + )] + pub stream_maxlen: usize, + #[serde(default = "ConfigGrpcAccounts::default_stream_data_key")] + pub stream_data_key: String, + + pub filters: Vec, +} + +impl ConfigGrpcAccounts { + pub fn default_stream() -> String { + REDIS_STREAM_ACCOUNTS.to_owned() + } + + pub const fn default_stream_maxlen() -> usize { + 100_000_000 + } + + pub fn default_stream_data_key() -> String { + "data".to_owned() + } +} + +#[derive(Debug, Deserialize)] +pub struct ConfigGrpcTransactions { + #[serde(default = "ConfigGrpcTransactions::default_stream")] + pub stream: String, + #[serde( + default = "ConfigGrpcTransactions::default_stream_maxlen", + deserialize_with = "deserialize_usize_str" + )] + pub stream_maxlen: usize, + #[serde(default = "ConfigGrpcTransactions::default_stream_data_key")] + pub stream_data_key: String, + + pub filters: Vec, +} + +impl ConfigGrpcTransactions { + pub fn default_stream() -> String { + REDIS_STREAM_TRANSACTIONS.to_owned() + } + + pub const fn default_stream_maxlen() -> usize { + 10_000_000 + } + + pub fn default_stream_data_key() -> String { + "data".to_owned() + } +} + +#[derive(Debug, Deserialize)] +pub struct ConfigGrpcRedis { + pub url: String, + #[serde( + default = "ConfigGrpcRedis::default_pipeline_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub pipeline_max_size: usize, + #[serde( + default = "ConfigGrpcRedis::default_pipeline_max_idle_ms", + deserialize_with = "deserialize_duration_str" + )] + pub pipeline_max_idle_ms: Duration, + #[serde( + default = "ConfigGrpcRedis::max_xadd_in_process", + deserialize_with = "deserialize_usize_str" + )] + pub max_xadd_in_process: usize, +} + +impl ConfigGrpcRedis { + pub const fn default_pipeline_max_size() -> usize { + 10 + } + + pub const fn default_pipeline_max_idle_ms() -> Duration { + Duration::from_millis(10) + } + + pub const fn max_xadd_in_process() -> usize { + 100 + } +} + +pub fn deserialize_duration_str<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + let ms = deserialize_usize_str(deserializer)?; + Ok(Duration::from_millis(ms as u64)) +} + +#[derive(Debug, Deserialize)] +pub struct ConfigIngester { + // +} diff --git a/nft_ingester2/src/grpc.rs b/nft_ingester2/src/grpc.rs new file mode 100644 index 000000000..a52bc2bc7 --- /dev/null +++ b/nft_ingester2/src/grpc.rs @@ -0,0 +1,147 @@ +use { + crate::{ + config::ConfigGrpc, prom::redis_xadd_status_inc, redis::metrics_xlen, util::create_shutdown, + }, + anyhow::Context, + futures::stream::StreamExt, + redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, + std::{sync::Arc, time::Duration}, + tokio::{ + signal::unix::SignalKind, + task::JoinSet, + time::{sleep, Instant}, + }, + yellowstone_grpc_client::GeyserGrpcClient, + yellowstone_grpc_proto::{prelude::subscribe_update::UpdateOneof, prost::Message}, +}; + +pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { + let config = Arc::new(config); + + // Connect to Redis + let client = redis::Client::open(config.redis.url.clone())?; + let connection = client.get_multiplexed_tokio_connection().await?; + + // Check stream length for the metrics + let jh_metrics_xlen = tokio::spawn({ + let connection = connection.clone(); + let streams = vec![ + config.accounts.stream.clone(), + config.transactions.stream.clone(), + ]; + async move { metrics_xlen(connection, &streams).await } + }); + tokio::pin!(jh_metrics_xlen); + + // Create gRPC client, subscribe and handle messages + let mut client = GeyserGrpcClient::build_from_shared(config.endpoint.clone())? + .x_token(config.x_token.clone())? + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(10)) + .connect() + .await + .context("failed to connect go gRPC")?; + let (mut _subscribe_tx, mut stream) = client.subscribe().await?; + + // recv-send loop + let mut shutdown = create_shutdown()?; + let mut pipe = redis::pipe(); + let mut pipe_accounts = 0; + let mut pipe_transactions = 0; + let deadline = sleep(config.redis.pipeline_max_idle_ms); + tokio::pin!(deadline); + let mut tasks = JoinSet::new(); + + let result = loop { + tokio::select! { + biased; + result = &mut jh_metrics_xlen => match result { + Ok(Ok(_)) => unreachable!(), + Ok(Err(error)) => break Err(error), + Err(error) => break Err(error.into()), + }, + signal = &mut shutdown => { + let signal = if signal == SignalKind::interrupt() { + "SIGINT" + } else if signal == SignalKind::terminate() { + "SIGTERM" + } else { + "UNKNOWN" + }; + tracing::warn!("{signal} received, waiting spawned tasks..."); + break Ok(()); + }, + msg = stream.next() => { + match msg { + Some(Ok(msg)) => match msg.update_oneof { + Some(UpdateOneof::Account(account)) => { + pipe.xadd_maxlen( + &config.accounts.stream, + StreamMaxlen::Approx(config.accounts.stream_maxlen), + "*", + &[(&config.accounts.stream_data_key, account.encode_to_vec())], + ); + pipe_accounts += 1; + } + Some(UpdateOneof::Slot(_)) => continue, + Some(UpdateOneof::Transaction(transaction)) => { + pipe.xadd_maxlen( + &config.transactions.stream, + StreamMaxlen::Approx(config.transactions.stream_maxlen), + "*", + &[(&config.transactions.stream_data_key, transaction.encode_to_vec())] + ); + pipe_transactions += 1; + } + Some(UpdateOneof::Block(_)) => continue, + Some(UpdateOneof::Ping(_)) => continue, + Some(UpdateOneof::Pong(_)) => continue, + Some(UpdateOneof::BlockMeta(_)) => continue, + Some(UpdateOneof::Entry(_)) => continue, + Some(UpdateOneof::TransactionStatus(_)) => continue, + None => break Err(anyhow::anyhow!("received invalid update gRPC message")), + }, + Some(Err(error)) => break Err(error.into()), + None => break Err(anyhow::anyhow!("geyser gRPC request is finished")), + }; + if pipe_accounts + pipe_transactions < config.redis.pipeline_max_size { + continue; + } + }, + _ = &mut deadline => {}, + }; + + let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); + let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); + let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); + deadline + .as_mut() + .reset(Instant::now() + config.redis.pipeline_max_idle_ms); + + tasks.spawn({ + let mut connection = connection.clone(); + let config = Arc::clone(&config); + async move { + let result: RedisResult = + pipe.atomic().query_async(&mut connection).await; + + let status = if result.is_ok() { Ok(()) } else { Err(()) }; + redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); + redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); + + Ok::<(), anyhow::Error>(()) + } + }); + while tasks.len() >= config.redis.max_xadd_in_process { + if let Some(result) = tasks.join_next().await { + result??; + } + } + }; + + while let Some(result) = tasks.join_next().await { + result??; + } + + result +} diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs new file mode 100644 index 000000000..475769af2 --- /dev/null +++ b/nft_ingester2/src/ingester.rs @@ -0,0 +1,5 @@ +use crate::config::ConfigIngester; + +pub async fn run(_config: ConfigIngester) -> anyhow::Result<()> { + todo!() +} diff --git a/nft_ingester2/src/main.rs b/nft_ingester2/src/main.rs new file mode 100644 index 000000000..7b72c18a9 --- /dev/null +++ b/nft_ingester2/src/main.rs @@ -0,0 +1,75 @@ +use { + crate::{ + config::{load as config_load, ConfigGrpc, ConfigIngester, ConfigPrometheus}, + prom::run_server as prometheus_run_server, + tracing::init as tracing_init, + }, + anyhow::Context, + clap::{Parser, Subcommand}, + std::net::SocketAddr, +}; + +mod config; +mod grpc; +mod ingester; +mod prom; +mod redis; +mod tracing; +mod util; +mod version; + +#[derive(Debug, Parser)] +#[clap(author, version)] +struct Args { + /// Path to config file + #[clap(short, long)] + config: String, + + /// Prometheus listen address + #[clap(long)] + prometheus: Option, + + #[command(subcommand)] + action: ArgsAction, +} + +#[derive(Debug, Clone, Subcommand)] +enum ArgsAction { + /// Subscribe on Geyser events using gRPC and send them to Redis + #[command(name = "grpc2redis")] + Grpc, + /// Run ingester process (process events from Redis) + #[command(name = "run")] + Ingester, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_init()?; + + let args = Args::parse(); + + // Run prometheus server + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse prometheus config from: {}", args.config))?; + if let Some(address) = args.prometheus.or(config.prometheus) { + prometheus_run_server(address)?; + } + + // Run grpc / ingester + match args.action { + ArgsAction::Grpc => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + grpc::run(config).await + } + ArgsAction::Ingester => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + ingester::run(config).await + } + } +} diff --git a/nft_ingester2/src/prom.rs b/nft_ingester2/src/prom.rs new file mode 100644 index 000000000..bf24a984a --- /dev/null +++ b/nft_ingester2/src/prom.rs @@ -0,0 +1,106 @@ +use { + crate::version::VERSION as VERSION_INFO, + hyper::{ + server::conn::AddrStream, + service::{make_service_fn, service_fn}, + Body, Request, Response, Server, StatusCode, + }, + prometheus::{IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder}, + std::{net::SocketAddr, sync::Once}, + tracing::{error, info}, +}; + +lazy_static::lazy_static! { + static ref REGISTRY: Registry = Registry::new(); + + static ref VERSION: IntCounterVec = IntCounterVec::new( + Opts::new("version", "Plugin version info"), + &["buildts", "git", "package", "proto", "rustc", "solana", "version"] + ).unwrap(); + + static ref REDIS_STREAM_LEN: IntGaugeVec = IntGaugeVec::new( + Opts::new("redis_stream_len", "Length of stream in Redis"), + &["stream"] + ).unwrap(); + + static ref REDIS_XADD_STATUS: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xadd_status", "Status of messages sent to Redis stream"), + &["stream", "status"] + ).unwrap(); +} + +pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { + macro_rules! register { + ($collector:ident) => { + REGISTRY + .register(Box::new($collector.clone())) + .expect("collector can't be registered"); + }; + } + register!(VERSION); + register!(REDIS_STREAM_LEN); + register!(REDIS_XADD_STATUS); + + VERSION + .with_label_values(&[ + VERSION_INFO.buildts, + VERSION_INFO.git, + VERSION_INFO.package, + VERSION_INFO.proto, + VERSION_INFO.rustc, + VERSION_INFO.solana, + VERSION_INFO.version, + ]) + .inc(); + }); + + let make_service = make_service_fn(move |_: &AddrStream| async move { + Ok::<_, hyper::Error>(service_fn(move |req: Request| async move { + let response = match req.uri().path() { + "/metrics" => metrics_handler(), + _ => not_found_handler(), + }; + Ok::<_, hyper::Error>(response) + })) + }); + let server = Server::try_bind(&address)?.serve(make_service); + info!("prometheus server started: {address:?}"); + tokio::spawn(async move { + if let Err(error) = server.await { + error!("prometheus server failed: {error:?}"); + } + }); + + Ok(()) +} + +fn metrics_handler() -> Response { + let metrics = TextEncoder::new() + .encode_to_string(®ISTRY.gather()) + .unwrap_or_else(|error| { + error!("could not encode custom metrics: {}", error); + String::new() + }); + Response::builder().body(Body::from(metrics)).unwrap() +} + +fn not_found_handler() -> Response { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::empty()) + .unwrap() +} + +pub fn redis_stream_len_set(stream: &str, len: usize) { + REDIS_STREAM_LEN + .with_label_values(&[stream]) + .set(len as i64); +} + +pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) { + REDIS_XADD_STATUS + .with_label_values(&[stream, if status.is_ok() { "success" } else { "failed" }]) + .inc_by(delta as u64); +} diff --git a/nft_ingester2/src/redis.rs b/nft_ingester2/src/redis.rs new file mode 100644 index 000000000..9e1c16ca4 --- /dev/null +++ b/nft_ingester2/src/redis.rs @@ -0,0 +1,25 @@ +use { + crate::prom::redis_stream_len_set, + redis::AsyncCommands, + std::convert::Infallible, + tokio::time::{sleep, Duration}, +}; + +pub async fn metrics_xlen( + mut connection: C, + streams: &[String], +) -> anyhow::Result { + loop { + let mut pipe = redis::pipe(); + for stream in streams { + pipe.xlen(stream); + } + let xlens: Vec = pipe.query_async(&mut connection).await?; + + for (stream, xlen) in streams.iter().zip(xlens.into_iter()) { + redis_stream_len_set(stream, xlen); + } + + sleep(Duration::from_millis(100)).await; + } +} diff --git a/nft_ingester2/src/tracing.rs b/nft_ingester2/src/tracing.rs new file mode 100644 index 000000000..36ba4db56 --- /dev/null +++ b/nft_ingester2/src/tracing.rs @@ -0,0 +1,32 @@ +use { + opentelemetry_sdk::trace::{self, Sampler}, + std::env, + tracing_subscriber::{filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}, +}; + +pub fn init() -> anyhow::Result<()> { + let open_tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_service_name(env::var("CARGO_PKG_NAME")?) + .with_auto_split_batch(true) + .with_trace_config(trace::config().with_sampler(Sampler::TraceIdRatioBased(0.25))) + .install_batch(opentelemetry_sdk::runtime::Tokio)?; + let jeager_layer = tracing_opentelemetry::layer().with_tracer(open_tracer); + + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + let is_atty = atty::is(atty::Stream::Stdout) && atty::is(atty::Stream::Stderr); + let io_layer = tracing_subscriber::fmt::layer().with_ansi(is_atty); + + let registry = tracing_subscriber::registry() + .with(jeager_layer) + .with(env_filter) + .with(io_layer); + + if env::var_os("RUST_LOG_JSON").is_some() { + let json_layer = tracing_subscriber::fmt::layer().json().flatten_event(true); + registry.with(json_layer).try_init() + } else { + registry.try_init() + } + .map_err(Into::into) +} diff --git a/nft_ingester2/src/util.rs b/nft_ingester2/src/util.rs new file mode 100644 index 000000000..6de97e42f --- /dev/null +++ b/nft_ingester2/src/util.rs @@ -0,0 +1,16 @@ +use { + futures::future::{BoxFuture, FutureExt}, + tokio::signal::unix::{signal, SignalKind}, +}; + +pub fn create_shutdown() -> anyhow::Result> { + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigterm = signal(SignalKind::terminate())?; + Ok(async move { + tokio::select! { + _ = sigint.recv() => SignalKind::interrupt(), + _ = sigterm.recv() => SignalKind::terminate(), + } + } + .boxed()) +} diff --git a/nft_ingester2/src/version.rs b/nft_ingester2/src/version.rs new file mode 100644 index 000000000..b9da62845 --- /dev/null +++ b/nft_ingester2/src/version.rs @@ -0,0 +1,22 @@ +use {serde::Serialize, std::env}; + +#[derive(Debug, Serialize)] +pub struct Version { + pub package: &'static str, + pub version: &'static str, + pub proto: &'static str, + pub solana: &'static str, + pub git: &'static str, + pub rustc: &'static str, + pub buildts: &'static str, +} + +pub const VERSION: Version = Version { + package: env!("CARGO_PKG_NAME"), + version: env!("CARGO_PKG_VERSION"), + proto: env!("YELLOWSTONE_GRPC_PROTO_VERSION"), + solana: env!("SOLANA_SDK_VERSION"), + git: env!("GIT_VERSION"), + rustc: env!("VERGEN_RUSTC_SEMVER"), + buildts: env!("VERGEN_BUILD_TIMESTAMP"), +}; From 076bffabca6bea8be61ec22a016401e3f475bedb Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Mon, 27 Nov 2023 21:29:06 -0500 Subject: [PATCH 14/72] add redis streaming for ingester --- nft_ingester2/README.md | 33 +++ nft_ingester2/config-run.yml | 19 ++ nft_ingester2/src/config.rs | 148 +++++++++++++- nft_ingester2/src/grpc.rs | 5 +- nft_ingester2/src/ingester.rs | 75 ++++++- nft_ingester2/src/main.rs | 1 + nft_ingester2/src/postgres.rs | 1 + nft_ingester2/src/prom.rs | 22 +- nft_ingester2/src/redis.rs | 375 +++++++++++++++++++++++++++++++++- 9 files changed, 654 insertions(+), 25 deletions(-) create mode 100644 nft_ingester2/README.md create mode 100644 nft_ingester2/config-run.yml create mode 100644 nft_ingester2/src/postgres.rs diff --git a/nft_ingester2/README.md b/nft_ingester2/README.md new file mode 100644 index 000000000..a2afa2226 --- /dev/null +++ b/nft_ingester2/README.md @@ -0,0 +1,33 @@ +## Dev process + +### geyser gRPC source + +Use [Triton One](https://triton.one/) provided endpoint or run own node with geyser plugin: https://github.com/rpcpool/yellowstone-grpc + +### Redis server + +``` +redis-server +``` + +### PostgreSQL server + +Run: + +``` +docker run -it --rm -e POSTGRES_PASSWORD=solana -e POSTGRES_USER=solana -e POSTGRES_DB=solana -p 5432:5432 postgres +``` + +Schema: + +> Also note: The migration `m20230224_093722_performance_improvements` needs to be commented out of the migration lib.rs in order for the Sea ORM `Relations` to generate correctly. + +``` +DATABASE_URL=postgres://solana:solana@localhost/solana INIT_FILE_PATH=init.sql cargo run -p migration --bin migration -- up +``` + +psql: + +``` +PGPASSWORD=solana psql -h localhost -U solana -d solana +``` diff --git a/nft_ingester2/config-run.yml b/nft_ingester2/config-run.yml new file mode 100644 index 000000000..281f6a1fd --- /dev/null +++ b/nft_ingester2/config-run.yml @@ -0,0 +1,19 @@ +prometheus: 127.0.0.1:8874 +redis: + url: redis://localhost:6379 + group: ingester + consumer: consumer # every new ingester instance should uniq name + streams: + - type: account # possible values: `account` and `transaction`, required for message decoding + stream: ACCOUNTS + data_key: data + - type: transaction + stream: TRANSACTIONS + data_key: data + xack_batch_max_size: 100 + xack_batch_max_idle_ms: 10 + xack_max_in_process: 100 + prefetch_queue_size: 1_000 # max number of messages available in the read queue for processing + xpending_max: 250 # used for reading pending messages + xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) + xreadgroup_max: 1_000 # applied per each stream in one request diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs index 14b45f247..c13ea7a18 100644 --- a/nft_ingester2/src/config.rs +++ b/nft_ingester2/src/config.rs @@ -11,6 +11,7 @@ use { pub const REDIS_STREAM_ACCOUNTS: &str = "ACCOUNTS"; pub const REDIS_STREAM_TRANSACTIONS: &str = "TRANSACTIONS"; +pub const REDIS_STREAM_DATA_KEY: &str = "data"; pub async fn load(path: impl AsRef + Copy) -> anyhow::Result where @@ -72,7 +73,7 @@ impl ConfigGrpcAccounts { } pub fn default_stream_data_key() -> String { - "data".to_owned() + REDIS_STREAM_DATA_KEY.to_owned() } } @@ -101,7 +102,7 @@ impl ConfigGrpcTransactions { } pub fn default_stream_data_key() -> String { - "data".to_owned() + REDIS_STREAM_DATA_KEY.to_owned() } } @@ -114,12 +115,13 @@ pub struct ConfigGrpcRedis { )] pub pipeline_max_size: usize, #[serde( - default = "ConfigGrpcRedis::default_pipeline_max_idle_ms", - deserialize_with = "deserialize_duration_str" + default = "ConfigGrpcRedis::default_pipeline_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "pipeline_max_idle_ms" )] - pub pipeline_max_idle_ms: Duration, + pub pipeline_max_idle: Duration, #[serde( - default = "ConfigGrpcRedis::max_xadd_in_process", + default = "ConfigGrpcRedis::default_max_xadd_in_process", deserialize_with = "deserialize_usize_str" )] pub max_xadd_in_process: usize, @@ -130,11 +132,11 @@ impl ConfigGrpcRedis { 10 } - pub const fn default_pipeline_max_idle_ms() -> Duration { + pub const fn default_pipeline_max_idle() -> Duration { Duration::from_millis(10) } - pub const fn max_xadd_in_process() -> usize { + pub const fn default_max_xadd_in_process() -> usize { 100 } } @@ -149,5 +151,133 @@ where #[derive(Debug, Deserialize)] pub struct ConfigIngester { - // + pub redis: ConfigIngesterRedis, +} + +#[derive(Debug, Deserialize)] +pub struct ConfigIngesterRedis { + pub url: String, + #[serde(default = "ConfigIngesterRedis::default_group")] + pub group: String, + #[serde(default = "ConfigIngesterRedis::default_consumer")] + pub consumer: String, + pub streams: Vec, + #[serde( + default = "ConfigIngesterRedis::default_prefetch_queue_size", + deserialize_with = "deserialize_usize_str" + )] + pub prefetch_queue_size: usize, + #[serde( + default = "ConfigIngesterRedis::default_xpending_max", + deserialize_with = "deserialize_usize_str" + )] + pub xpending_max: usize, + #[serde(default = "ConfigIngesterRedis::default_xpending_only")] + pub xpending_only: bool, + #[serde( + default = "ConfigIngesterRedis::default_xreadgroup_max", + deserialize_with = "deserialize_usize_str" + )] + pub xreadgroup_max: usize, +} + +impl ConfigIngesterRedis { + pub fn default_group() -> String { + "ingester".to_owned() + } + + pub fn default_consumer() -> String { + "consumer".to_owned() + } + + pub const fn default_prefetch_queue_size() -> usize { + 1_000 + } + + pub const fn default_xpending_max() -> usize { + 100 + } + + pub const fn default_xpending_only() -> bool { + false + } + + pub const fn default_xreadgroup_max() -> usize { + 1_000 + } +} + +#[derive(Debug, Clone)] +pub struct ConfigIngesterRedisStream { + pub stream_type: ConfigIngesterRedisStreamType, + pub stream: String, + pub data_key: String, + pub xack_batch_max_size: usize, + pub xack_batch_max_idle: Duration, + pub xack_max_in_process: usize, +} + +impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + #[derive(Debug, Deserialize)] + struct Raw { + #[serde(rename = "type")] + pub stream_type: ConfigIngesterRedisStreamType, + pub stream: Option, + pub data_key: Option, + #[serde( + default = "default_xack_batch_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub xack_batch_max_size: usize, + #[serde( + default = "default_xack_batch_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "xack_batch_max_idle_ms" + )] + pub xack_batch_max_idle: Duration, + #[serde( + default = "default_xack_max_in_process", + deserialize_with = "deserialize_usize_str" + )] + pub xack_max_in_process: usize, + } + + const fn default_xack_batch_max_size() -> usize { + 100 + } + + const fn default_xack_batch_max_idle() -> Duration { + Duration::from_millis(10) + } + + const fn default_xack_max_in_process() -> usize { + 100 + } + + let raw = Raw::deserialize(deserializer)?; + Ok(Self { + stream_type: raw.stream_type, + stream: raw.stream.unwrap_or_else(|| match raw.stream_type { + ConfigIngesterRedisStreamType::Account => REDIS_STREAM_ACCOUNTS.to_owned(), + ConfigIngesterRedisStreamType::Transaction => REDIS_STREAM_TRANSACTIONS.to_owned(), + }), + data_key: raw + .data_key + .unwrap_or_else(|| REDIS_STREAM_DATA_KEY.to_owned()), + xack_batch_max_size: raw.xack_batch_max_size, + xack_batch_max_idle: raw.xack_batch_max_idle, + xack_max_in_process: raw.xack_max_in_process, + }) + } +} + +#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ConfigIngesterRedisStreamType { + Account, + Transaction, } diff --git a/nft_ingester2/src/grpc.rs b/nft_ingester2/src/grpc.rs index a52bc2bc7..aebb88b8a 100644 --- a/nft_ingester2/src/grpc.rs +++ b/nft_ingester2/src/grpc.rs @@ -48,13 +48,12 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let mut pipe = redis::pipe(); let mut pipe_accounts = 0; let mut pipe_transactions = 0; - let deadline = sleep(config.redis.pipeline_max_idle_ms); + let deadline = sleep(config.redis.pipeline_max_idle); tokio::pin!(deadline); let mut tasks = JoinSet::new(); let result = loop { tokio::select! { - biased; result = &mut jh_metrics_xlen => match result { Ok(Ok(_)) => unreachable!(), Ok(Err(error)) => break Err(error), @@ -116,7 +115,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); deadline .as_mut() - .reset(Instant::now() + config.redis.pipeline_max_idle_ms); + .reset(Instant::now() + config.redis.pipeline_max_idle); tasks.spawn({ let mut connection = connection.clone(); diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 475769af2..1e02e03ed 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -1,5 +1,74 @@ -use crate::config::ConfigIngester; +use { + crate::{ + config::ConfigIngester, + redis::{metrics_xlen, RedisStream}, + util::create_shutdown, + }, + futures::future::{Fuse, FusedFuture, FutureExt}, + tokio::signal::unix::SignalKind, +}; -pub async fn run(_config: ConfigIngester) -> anyhow::Result<()> { - todo!() +pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { + println!("{:#?}", config); + + // connect to Redis + let client = redis::Client::open(config.redis.url.clone())?; + let connection = client.get_multiplexed_tokio_connection().await?; + + // check stream length for the metrics in spawned task + let jh_metrics_xlen = tokio::spawn({ + let connection = connection.clone(); + let streams = config + .redis + .streams + .iter() + .map(|config| config.stream.clone()) + .collect::>(); + async move { metrics_xlen(connection, &streams).await } + }); + tokio::pin!(jh_metrics_xlen); + + // create redis stream reader + let (mut redis_messages, redis_tasks) = RedisStream::new(config.redis, connection).await?; + let redis_tasks_fut = Fuse::terminated(); + tokio::pin!(redis_tasks_fut); + redis_tasks_fut.set(redis_tasks.fuse()); + + // read messages in the loop + let mut shutdown = create_shutdown()?; + let result = loop { + tokio::select! { + result = &mut jh_metrics_xlen => match result { + Ok(Ok(_)) => unreachable!(), + Ok(Err(error)) => break Err(error), + Err(error) => break Err(error.into()), + }, + signal = &mut shutdown => { + let signal = if signal == SignalKind::interrupt() { + "SIGINT" + } else if signal == SignalKind::terminate() { + "SIGTERM" + } else { + "UNKNOWN" + }; + tracing::warn!("{signal} received, waiting spawned tasks..."); + break Ok(()); + }, + result = &mut redis_tasks_fut => break result, + msg = redis_messages.recv() => match msg { + Some(msg) => { + // TODO: process messages here + msg.ack()?; + } + None => break Ok(()), + } + }; + }; + + redis_messages.shutdown(); + if !redis_tasks_fut.is_terminated() { + redis_tasks_fut.await?; + } + + result } diff --git a/nft_ingester2/src/main.rs b/nft_ingester2/src/main.rs index 7b72c18a9..0bd1f9b8d 100644 --- a/nft_ingester2/src/main.rs +++ b/nft_ingester2/src/main.rs @@ -12,6 +12,7 @@ use { mod config; mod grpc; mod ingester; +mod postgres; mod prom; mod redis; mod tracing; diff --git a/nft_ingester2/src/postgres.rs b/nft_ingester2/src/postgres.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/nft_ingester2/src/postgres.rs @@ -0,0 +1 @@ + diff --git a/nft_ingester2/src/prom.rs b/nft_ingester2/src/prom.rs index bf24a984a..b6794b615 100644 --- a/nft_ingester2/src/prom.rs +++ b/nft_ingester2/src/prom.rs @@ -18,8 +18,8 @@ lazy_static::lazy_static! { &["buildts", "git", "package", "proto", "rustc", "solana", "version"] ).unwrap(); - static ref REDIS_STREAM_LEN: IntGaugeVec = IntGaugeVec::new( - Opts::new("redis_stream_len", "Length of stream in Redis"), + static ref REDIS_XLEN_TOTAL: IntGaugeVec = IntGaugeVec::new( + Opts::new("redis_xlen_total", "Length of stream in Redis"), &["stream"] ).unwrap(); @@ -27,6 +27,11 @@ lazy_static::lazy_static! { Opts::new("redis_xadd_status", "Status of messages sent to Redis stream"), &["stream", "status"] ).unwrap(); + + static ref REDIS_XACK_TOTAL: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xack_total", "Total number of processed messages"), + &["stream"] + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -40,8 +45,9 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { }; } register!(VERSION); - register!(REDIS_STREAM_LEN); + register!(REDIS_XLEN_TOTAL); register!(REDIS_XADD_STATUS); + register!(REDIS_XACK_TOTAL); VERSION .with_label_values(&[ @@ -93,8 +99,8 @@ fn not_found_handler() -> Response { .unwrap() } -pub fn redis_stream_len_set(stream: &str, len: usize) { - REDIS_STREAM_LEN +pub fn redis_xlen_set(stream: &str, len: usize) { + REDIS_XLEN_TOTAL .with_label_values(&[stream]) .set(len as i64); } @@ -104,3 +110,9 @@ pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) .with_label_values(&[stream, if status.is_ok() { "success" } else { "failed" }]) .inc_by(delta as u64); } + +pub fn redis_xack_inc(stream: &str, delta: usize) { + REDIS_XACK_TOTAL + .with_label_values(&[stream]) + .inc_by(delta as u64) +} diff --git a/nft_ingester2/src/redis.rs b/nft_ingester2/src/redis.rs index 9e1c16ca4..47f4584f3 100644 --- a/nft_ingester2/src/redis.rs +++ b/nft_ingester2/src/redis.rs @@ -1,8 +1,34 @@ use { - crate::prom::redis_stream_len_set, - redis::AsyncCommands, - std::convert::Infallible, - tokio::time::{sleep, Duration}, + crate::{ + config::{ConfigIngesterRedis, ConfigIngesterRedisStreamType}, + prom::{redis_xack_inc, redis_xlen_set}, + }, + futures::future::{BoxFuture, FutureExt}, + redis::{ + aio::MultiplexedConnection, + streams::{ + StreamClaimReply, StreamId, StreamKey, StreamPendingCountReply, StreamReadOptions, + StreamReadReply, + }, + AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, + }, + std::{ + collections::HashMap, + convert::Infallible, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + }, + tokio::{ + sync::mpsc, + task::JoinSet, + time::{sleep, Duration, Instant}, + }, + yellowstone_grpc_proto::{ + prelude::{SubscribeUpdateAccount, SubscribeUpdateTransaction}, + prost::Message, + }, }; pub async fn metrics_xlen( @@ -17,9 +43,348 @@ pub async fn metrics_xlen( let xlens: Vec = pipe.query_async(&mut connection).await?; for (stream, xlen) in streams.iter().zip(xlens.into_iter()) { - redis_stream_len_set(stream, xlen); + redis_xlen_set(stream, xlen); } sleep(Duration::from_millis(100)).await; } } + +pub async fn xgroup_create( + connection: &mut C, + name: &str, + group: &str, + consumer: &str, +) -> anyhow::Result<()> { + let result: RedisResult = connection.xgroup_create_mkstream(name, group, "0").await; + if let Err(error) = result { + if !(error.kind() == RedisErrorKind::ExtensionError + && error.detail() == Some("Consumer Group name already exists") + && error.code() == Some("BUSYGROUP")) + { + return Err(error.into()); + } + } + + // XGROUP CREATECONSUMER key group consumer + redis::cmd("XGROUP") + .arg("CREATECONSUMER") + .arg(name) + .arg(group) + .arg(consumer) + .query_async(connection) + .await?; + + Ok(()) +} + +#[derive(Debug)] +struct RedisStreamInfo { + group: String, + consumer: String, + stream_name: String, + stream_type: ConfigIngesterRedisStreamType, + stream_data_key: String, + xack_batch_max_size: usize, + xack_batch_max_idle: Duration, + xack_max_in_process: usize, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum RedisStreamMessage { + Account(SubscribeUpdateAccount), + Transaction(SubscribeUpdateTransaction), +} + +#[derive(Debug)] +pub struct RedisStreamMessageInfo { + id: String, + data: RedisStreamMessage, + ack_tx: mpsc::UnboundedSender, +} + +impl RedisStreamMessageInfo { + fn parse( + stream: &RedisStreamInfo, + StreamId { id, map }: StreamId, + ack_tx: mpsc::UnboundedSender, + ) -> anyhow::Result { + let data = match map.get(&stream.stream_data_key) { + Some(RedisValue::Data(vec)) => match stream.stream_type { + ConfigIngesterRedisStreamType::Account => { + RedisStreamMessage::Account(SubscribeUpdateAccount::decode(vec.as_ref())?) + } + ConfigIngesterRedisStreamType::Transaction => RedisStreamMessage::Transaction( + SubscribeUpdateTransaction::decode(vec.as_ref())?, + ), + }, + Some(_) => anyhow::bail!( + "invalid data (key: {:?}) from stream {:?}", + stream.stream_data_key, + stream.stream_name + ), + None => anyhow::bail!( + "failed to get data (key: {:?}) from stream {:?}", + stream.stream_data_key, + stream.stream_name + ), + }; + Ok(Self { id, data, ack_tx }) + } + + pub fn ack(self) -> anyhow::Result<()> { + self.ack_tx + .send(self.id) + .map_err(|_error| anyhow::anyhow!("failed to send message to ack channel",)) + } +} + +#[derive(Debug)] +pub struct RedisStream { + shutdown: Arc, + messages_rx: mpsc::Receiver, +} + +impl RedisStream { + pub async fn new( + config: ConfigIngesterRedis, + mut connection: MultiplexedConnection, + ) -> anyhow::Result<(Self, BoxFuture<'static, anyhow::Result<()>>)> { + // create group with consumer per stream + for stream in config.streams.iter() { + xgroup_create( + &mut connection, + &stream.stream, + &config.group, + &config.consumer, + ) + .await?; + } + + // shutdown flag + let shutdown = Arc::new(AtomicBool::new(false)); + + // create stream info wrapped by Arc + let mut ack_tasks = vec![]; + let streams = config + .streams + .iter() + .map(|stream| { + let (ack_tx, ack_rx) = mpsc::unbounded_channel(); + let info = Arc::new(RedisStreamInfo { + group: config.group.clone(), + consumer: config.consumer.clone(), + stream_name: stream.stream.clone(), + stream_type: stream.stream_type, + stream_data_key: stream.data_key.clone(), + xack_batch_max_size: stream.xack_batch_max_size, + xack_batch_max_idle: stream.xack_batch_max_idle, + xack_max_in_process: stream.xack_max_in_process, + }); + ack_tasks.push((Arc::clone(&info), ack_rx)); + (stream.stream.clone(), (ack_tx, info)) + }) + .collect::>(); + + // spawn ack tasks + let mut tasks = ack_tasks + .into_iter() + .map(|(stream, ack_rx)| { + let connection = connection.clone(); + tokio::spawn(async move { Self::run_ack(stream, connection, ack_rx).await }) + }) + .collect::>(); + + // spawn prefetch task + let (messages_tx, messages_rx) = mpsc::channel(config.prefetch_queue_size); + tasks.push(tokio::spawn({ + let shutdown = Arc::clone(&shutdown); + async move { Self::run_prefetch(config, streams, connection, messages_tx, shutdown).await } + })); + + // merge spawn tasks + let spawned_tasks = async move { + for task in tasks.into_iter() { + task.await??; + } + Ok::<(), anyhow::Error>(()) + } + .boxed(); + + Ok(( + Self { + shutdown, + messages_rx, + }, + spawned_tasks, + )) + } + + pub async fn recv(&mut self) -> Option { + self.messages_rx.recv().await + } + + pub fn shutdown(self) { + self.shutdown.store(true, Ordering::Relaxed); + } + + async fn run_prefetch( + config: ConfigIngesterRedis, + streams: HashMap, Arc)>, + mut connection: MultiplexedConnection, + messages_tx: mpsc::Sender, + shutdown: Arc, + ) -> anyhow::Result<()> { + // read pending first + for (ack_tx, stream) in streams.values() { + let mut start = "-".to_owned(); + while !shutdown.load(Ordering::Relaxed) { + let StreamPendingCountReply { ids: pending_ids } = redis::cmd("XPENDING") + .arg(&stream.stream_name) + .arg(&stream.group) + .arg(&start) + .arg("+") + .arg(config.xpending_max) + .arg(&stream.consumer) // we can't use `xpending_count` because it doesn't support `consumer` filter + .query_async(&mut connection) + .await?; + + // drop first item if we do not start from the beginning + let used_ids = if start == "-" { 0.. } else { 1.. }; + let ids_str = pending_ids[used_ids] + .iter() + .map(|pending| pending.id.as_str()) + .collect::>(); + + // check that we fetched all pendings and update start + match pending_ids.last() { + Some(id) => { + if id.id == start { + break; + } else { + start = id.id.clone(); + } + } + None => break, + } + + // read pending keys + let StreamClaimReply { ids: pendings } = connection + .xclaim( + &stream.stream_name, + &stream.group, + &stream.consumer, + 0, + &ids_str, + ) + .await?; + for pending in pendings { + let item = RedisStreamMessageInfo::parse(stream, pending, ack_tx.clone())?; + messages_tx.send(item).await.map_err(|_error| { + anyhow::anyhow!("failed to send item to prefetch channel") + })?; + } + } + } + + // exit if need to handle only pending + if config.xpending_only { + return Ok(()); + } + + let streams_keys = streams.keys().map(|name| name.as_str()).collect::>(); + let streams_ids = (0..streams_keys.len()).map(|_| ">").collect::>(); + while !shutdown.load(Ordering::Relaxed) { + let opts = StreamReadOptions::default() + .count(config.xreadgroup_max) + .group(&config.group, &config.consumer); + let results: StreamReadReply = connection + .xread_options(&streams_keys, &streams_ids, &opts) + .await?; + if results.keys.is_empty() { + sleep(Duration::from_millis(5)).await; + continue; + } + + for StreamKey { key, ids } in results.keys { + let (ack_tx, stream) = match streams.get(&key) { + Some(value) => value, + None => anyhow::bail!("unknown stream: {:?}", key), + }; + + for id in ids { + let item = RedisStreamMessageInfo::parse(stream, id, ack_tx.clone())?; + messages_tx.send(item).await.map_err(|_error| { + anyhow::anyhow!("failed to send item to prefetch channel") + })?; + } + } + } + + Ok(()) + } + + async fn run_ack( + stream: Arc, + connection: MultiplexedConnection, + mut ack_rx: mpsc::UnboundedReceiver, + ) -> anyhow::Result<()> { + let mut ids = vec![]; + let deadline = sleep(stream.xack_batch_max_idle); + tokio::pin!(deadline); + let mut tasks = JoinSet::new(); + + let result = loop { + let terminated = tokio::select! { + msg = ack_rx.recv() => match msg { + Some(msg) => { + ids.push(msg); + if ids.len() < stream.xack_batch_max_size { + continue; + } + false + } + None => true, + }, + _ = &mut deadline => false, + }; + + let ids = std::mem::take(&mut ids); + deadline + .as_mut() + .reset(Instant::now() + stream.xack_batch_max_idle); + if !ids.is_empty() { + tasks.spawn({ + let stream = Arc::clone(&stream); + let mut connection = connection.clone(); + async move { + redis::pipe() + .atomic() + .xack(&stream.stream_name, &stream.group, &ids) + .xdel(&stream.stream_name, &ids) + .query_async(&mut connection) + .await?; + redis_xack_inc(&stream.stream_name, ids.len()); + Ok::<(), anyhow::Error>(()) + } + }); + while tasks.len() >= stream.xack_max_in_process { + if let Some(result) = tasks.join_next().await { + result??; + } + } + } + + if terminated { + break Ok(()); + } + }; + + while let Some(result) = tasks.join_next().await { + result??; + } + + result + } +} From d8673cf38f37f5eeac6d711b1b06823dfa483844 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Tue, 28 Nov 2023 21:57:16 -0500 Subject: [PATCH 15/72] create pg pool --- Cargo.lock | 1 + nft_ingester2/Cargo.toml | 1 + nft_ingester2/config-run.yml | 6 +++++- nft_ingester2/src/config.rs | 26 ++++++++++++++++++++++++++ nft_ingester2/src/ingester.rs | 17 +++++++++++++++++ nft_ingester2/src/postgres.rs | 30 ++++++++++++++++++++++++++++++ nft_ingester2/src/prom.rs | 21 +++++++++++++++++++++ nft_ingester2/src/redis.rs | 6 +++--- 8 files changed, 104 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index debbc5287..1d35a7e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3738,6 +3738,7 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "opentelemetry_sdk", + "program_transformers", "prometheus", "redis 0.25.3", "serde", diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml index 5a4e0f142..b58975c1c 100644 --- a/nft_ingester2/Cargo.toml +++ b/nft_ingester2/Cargo.toml @@ -16,6 +16,7 @@ lazy_static = { workspace = true } opentelemetry = { workspace = true } opentelemetry-jaeger = { workspace = true, features = ["rt-tokio"] } opentelemetry_sdk = { workspace = true, features = ["trace"] } +program_transformers = { workspace = true } prometheus = { workspace = true } redis = { workspace = true, features = ["tokio-comp", "tokio-native-tls-comp"] } serde = { workspace = true } diff --git a/nft_ingester2/config-run.yml b/nft_ingester2/config-run.yml index 281f6a1fd..3acd4ea24 100644 --- a/nft_ingester2/config-run.yml +++ b/nft_ingester2/config-run.yml @@ -2,7 +2,7 @@ prometheus: 127.0.0.1:8874 redis: url: redis://localhost:6379 group: ingester - consumer: consumer # every new ingester instance should uniq name + consumer: consumer # every new ingester instance should have uniq name streams: - type: account # possible values: `account` and `transaction`, required for message decoding stream: ACCOUNTS @@ -17,3 +17,7 @@ redis: xpending_max: 250 # used for reading pending messages xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) xreadgroup_max: 1_000 # applied per each stream in one request +postgres: + url: postgres://solana:solana@localhost/solana + min_connections: 10 + max_connections: 25 diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs index c13ea7a18..a79c167d4 100644 --- a/nft_ingester2/src/config.rs +++ b/nft_ingester2/src/config.rs @@ -152,6 +152,7 @@ where #[derive(Debug, Deserialize)] pub struct ConfigIngester { pub redis: ConfigIngesterRedis, + pub postgres: ConfigIngesterPostgres, } #[derive(Debug, Deserialize)] @@ -281,3 +282,28 @@ pub enum ConfigIngesterRedisStreamType { Account, Transaction, } + +#[derive(Debug, Deserialize)] +pub struct ConfigIngesterPostgres { + pub url: String, + #[serde( + default = "ConfigIngesterPostgres::default_min_connections", + deserialize_with = "deserialize_usize_str" + )] + pub min_connections: usize, + #[serde( + default = "ConfigIngesterPostgres::default_max_connections", + deserialize_with = "deserialize_usize_str" + )] + pub max_connections: usize, +} + +impl ConfigIngesterPostgres { + pub const fn default_min_connections() -> usize { + 10 + } + + pub const fn default_max_connections() -> usize { + 25 + } +} diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 1e02e03ed..054be0d5a 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -1,10 +1,13 @@ use { crate::{ config::ConfigIngester, + postgres::{create_pool as pg_create_pool, metrics_pgpool}, redis::{metrics_xlen, RedisStream}, util::create_shutdown, }, futures::future::{Fuse, FusedFuture, FutureExt}, + program_transformers::ProgramTransformer, + std::sync::Arc, tokio::signal::unix::SignalKind, }; @@ -28,6 +31,13 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { }); tokio::pin!(jh_metrics_xlen); + // open connection to postgres + let pgpool = pg_create_pool(config.postgres).await?; + tokio::spawn({ + let pgpool = Arc::clone(&pgpool); + async move { metrics_pgpool(pgpool).await } + }); + // create redis stream reader let (mut redis_messages, redis_tasks) = RedisStream::new(config.redis, connection).await?; let redis_tasks_fut = Fuse::terminated(); @@ -69,6 +79,13 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { if !redis_tasks_fut.is_terminated() { redis_tasks_fut.await?; } + pgpool.close().await; result } + +async fn run_program_transformers() -> anyhow::Result<()> { + // let pt_accounts = ProgramTransformer::new() + + todo!() +} diff --git a/nft_ingester2/src/postgres.rs b/nft_ingester2/src/postgres.rs index 8b1378917..6f7bc3c0d 100644 --- a/nft_ingester2/src/postgres.rs +++ b/nft_ingester2/src/postgres.rs @@ -1 +1,31 @@ +use { + crate::{ + config::ConfigIngesterPostgres, + prom::{pgpool_connections_set, PgpoolConnectionsKind}, + }, + sqlx::{ + postgres::{PgConnectOptions, PgPoolOptions}, + PgPool, + }, + std::sync::Arc, + tokio::time::{sleep, Duration}, +}; +pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result> { + let options: PgConnectOptions = config.url.parse()?; + PgPoolOptions::new() + .min_connections(config.min_connections.try_into()?) + .max_connections(config.max_connections.try_into()?) + .connect_with(options) + .await + .map(Arc::new) + .map_err(Into::into) +} + +pub async fn metrics_pgpool(pgpool: Arc) { + loop { + pgpool_connections_set(PgpoolConnectionsKind::Total, pgpool.size() as usize); + pgpool_connections_set(PgpoolConnectionsKind::Idle, pgpool.num_idle()); + sleep(Duration::from_millis(100)).await; + } +} diff --git a/nft_ingester2/src/prom.rs b/nft_ingester2/src/prom.rs index b6794b615..1b048671e 100644 --- a/nft_ingester2/src/prom.rs +++ b/nft_ingester2/src/prom.rs @@ -32,6 +32,11 @@ lazy_static::lazy_static! { Opts::new("redis_xack_total", "Total number of processed messages"), &["stream"] ).unwrap(); + + static ref PGPOOL_CONNECTIONS_TOTAL: IntGaugeVec = IntGaugeVec::new( + Opts::new("pgpool_connections_total", "Total number of connections in Postgres Pool"), + &["kind"] + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -48,6 +53,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(REDIS_XLEN_TOTAL); register!(REDIS_XADD_STATUS); register!(REDIS_XACK_TOTAL); + register!(PGPOOL_CONNECTIONS_TOTAL); VERSION .with_label_values(&[ @@ -116,3 +122,18 @@ pub fn redis_xack_inc(stream: &str, delta: usize) { .with_label_values(&[stream]) .inc_by(delta as u64) } + +#[derive(Debug, Clone, Copy)] +pub enum PgpoolConnectionsKind { + Total, + Idle, +} + +pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { + PGPOOL_CONNECTIONS_TOTAL + .with_label_values(&[match kind { + PgpoolConnectionsKind::Total => "total", + PgpoolConnectionsKind::Idle => "idle", + }]) + .set(size as i64) +} diff --git a/nft_ingester2/src/redis.rs b/nft_ingester2/src/redis.rs index 47f4584f3..554f02c6e 100644 --- a/nft_ingester2/src/redis.rs +++ b/nft_ingester2/src/redis.rs @@ -136,7 +136,7 @@ impl RedisStreamMessageInfo { pub fn ack(self) -> anyhow::Result<()> { self.ack_tx .send(self.id) - .map_err(|_error| anyhow::anyhow!("failed to send message to ack channel",)) + .map_err(|_error| anyhow::anyhow!("failed to send message to ack channel")) } } @@ -187,7 +187,7 @@ impl RedisStream { }) .collect::>(); - // spawn ack tasks + // spawn xack tasks let mut tasks = ack_tasks .into_iter() .map(|(stream, ack_rx)| { @@ -203,7 +203,7 @@ impl RedisStream { async move { Self::run_prefetch(config, streams, connection, messages_tx, shutdown).await } })); - // merge spawn tasks + // merge spawned xack / prefetch tasks let spawned_tasks = async move { for task in tasks.into_iter() { task.await??; From c93adffc6e5dde712e1469af06b651ea00ce6cbe Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Tue, 28 Nov 2023 23:01:47 -0500 Subject: [PATCH 16/72] parse incoming message from redis --- Cargo.lock | 1 + nft_ingester2/Cargo.toml | 8 ++ nft_ingester2/config-run.yml | 3 + nft_ingester2/src/config.rs | 22 +++++ nft_ingester2/src/ingester.rs | 171 +++++++++++++++++++++++++++++----- nft_ingester2/src/postgres.rs | 6 +- nft_ingester2/src/prom.rs | 36 ++++++- nft_ingester2/src/redis.rs | 90 +++++++++++++++--- 8 files changed, 297 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d35a7e83..fd708440f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3744,6 +3744,7 @@ dependencies = [ "serde", "serde_yaml", "solana-sdk", + "sqlx", "tokio", "tracing", "tracing-opentelemetry", diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml index b58975c1c..455f2902d 100644 --- a/nft_ingester2/Cargo.toml +++ b/nft_ingester2/Cargo.toml @@ -8,6 +8,14 @@ publish = { workspace = true } [dependencies] anyhow = { workspace = true } atty = { workspace = true } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } clap = { workspace = true, features = ["cargo", "derive"] } futures = { workspace = true } hyper = { workspace = true, features = ["server"] } diff --git a/nft_ingester2/config-run.yml b/nft_ingester2/config-run.yml index 3acd4ea24..19a1b3f84 100644 --- a/nft_ingester2/config-run.yml +++ b/nft_ingester2/config-run.yml @@ -21,3 +21,6 @@ postgres: url: postgres://solana:solana@localhost/solana min_connections: 10 max_connections: 25 +program_transformer: + transactions_cl_audits: false + max_tasks_in_process: 100 diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs index a79c167d4..003985f27 100644 --- a/nft_ingester2/src/config.rs +++ b/nft_ingester2/src/config.rs @@ -153,6 +153,7 @@ where pub struct ConfigIngester { pub redis: ConfigIngesterRedis, pub postgres: ConfigIngesterPostgres, + pub program_transformer: ConfigIngesterProgramTransformer, } #[derive(Debug, Deserialize)] @@ -307,3 +308,24 @@ impl ConfigIngesterPostgres { 25 } } + +#[derive(Debug, Deserialize)] +pub struct ConfigIngesterProgramTransformer { + #[serde(default = "ConfigIngesterProgramTransformer::default_transactions_cl_audits")] + pub transactions_cl_audits: bool, + #[serde( + default = "ConfigIngesterProgramTransformer::default_max_tasks_in_process", + deserialize_with = "deserialize_usize_str" + )] + pub max_tasks_in_process: usize, +} + +impl ConfigIngesterProgramTransformer { + pub const fn default_transactions_cl_audits() -> bool { + false + } + + pub const fn default_max_tasks_in_process() -> usize { + 100 + } +} diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 054be0d5a..0d7d43478 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -2,13 +2,28 @@ use { crate::{ config::ConfigIngester, postgres::{create_pool as pg_create_pool, metrics_pgpool}, - redis::{metrics_xlen, RedisStream}, + prom::{ + program_transformer_task_status_inc, program_transformer_tasks_total_set, + ProgramTransformerTaskStatusKind, + }, + redis::{metrics_xlen, ProgramTransformerInfo, RedisStream}, util::create_shutdown, }, - futures::future::{Fuse, FusedFuture, FutureExt}, - program_transformers::ProgramTransformer, - std::sync::Arc, - tokio::signal::unix::SignalKind, + futures::future::{pending, BoxFuture, FusedFuture, FutureExt}, + program_transformers::{ + error::ProgramTransformerError, DownloadMetadataInfo, DownloadMetadataNotifier, + ProgramTransformer, + }, + std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + tokio::{ + signal::unix::SignalKind, + task::JoinSet, + time::{sleep, Duration}, + }, + tracing::warn, }; pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { @@ -34,20 +49,55 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // open connection to postgres let pgpool = pg_create_pool(config.postgres).await?; tokio::spawn({ - let pgpool = Arc::clone(&pgpool); + let pgpool = pgpool.clone(); async move { metrics_pgpool(pgpool).await } }); // create redis stream reader - let (mut redis_messages, redis_tasks) = RedisStream::new(config.redis, connection).await?; - let redis_tasks_fut = Fuse::terminated(); + let (mut redis_messages, redis_tasks_fut) = RedisStream::new(config.redis, connection).await?; tokio::pin!(redis_tasks_fut); - redis_tasks_fut.set(redis_tasks.fuse()); - // read messages in the loop + // program transforms related + let pt_accounts = Arc::new(ProgramTransformer::new( + pgpool.clone(), + create_notifier(), + false, + )); + let pt_transactions = Arc::new(ProgramTransformer::new( + pgpool.clone(), + create_notifier(), + config.program_transformer.transactions_cl_audits, + )); + let pt_max_tasks_in_process = config.program_transformer.max_tasks_in_process; + let mut pt_tasks = JoinSet::new(); + let pt_tasks_len = Arc::new(AtomicUsize::new(0)); + tokio::spawn({ + let pt_tasks_len = Arc::clone(&pt_tasks_len); + async move { + loop { + program_transformer_tasks_total_set(pt_tasks_len.load(Ordering::Relaxed)); + sleep(Duration::from_millis(100)).await; + } + } + }); + + // read and process messages in the loop let mut shutdown = create_shutdown()?; let result = loop { - tokio::select! { + pt_tasks_len.store(pt_tasks.len(), Ordering::Relaxed); + + let redis_messages_recv = if pt_tasks.len() == pt_max_tasks_in_process { + pending().boxed() + } else { + redis_messages.recv().boxed() + }; + let pt_tasks_next = if pt_tasks.is_empty() { + pending().boxed() + } else { + pt_tasks.join_next().boxed() + }; + + let msg = tokio::select! { result = &mut jh_metrics_xlen => match result { Ok(Ok(_)) => unreachable!(), Ok(Err(error)) => break Err(error), @@ -61,21 +111,94 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { } else { "UNKNOWN" }; - tracing::warn!("{signal} received, waiting spawned tasks..."); + warn!("{signal} received, waiting spawned tasks..."); break Ok(()); }, result = &mut redis_tasks_fut => break result, - msg = redis_messages.recv() => match msg { - Some(msg) => { - // TODO: process messages here - msg.ack()?; - } + msg = redis_messages_recv => match msg { + Some(msg) => msg, None => break Ok(()), + }, + result = pt_tasks_next => { + if let Some(result) = result { + result??; + } + continue; } }; + + pt_tasks.spawn({ + let pt_accounts = Arc::clone(&pt_accounts); + let pt_transactions = Arc::clone(&pt_transactions); + async move { + let result = match &msg.get_data() { + ProgramTransformerInfo::Account(account) => { + pt_accounts.handle_account_update(account).await + } + ProgramTransformerInfo::Transaction(transaction) => { + pt_transactions.handle_transaction(transaction).await + } + }; + + macro_rules! log_or_bail { + ($action:path, $msg:expr, $error:ident) => { + match msg.get_data() { + ProgramTransformerInfo::Account(account) => { + $action!("{} account {}: {:?}", $msg, account.pubkey, $error) + } + ProgramTransformerInfo::Transaction(transaction) => { + $action!( + "{} transaction {}: {:?}", + $msg, + transaction.signature, + $error + ) + } + } + }; + } + + match result { + Ok(()) => program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::Success, + ), + Err(ProgramTransformerError::NotImplemented) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::NotImplemented, + ) + } + Err(ProgramTransformerError::DeserializationError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::DeserializationError, + ); + log_or_bail!(warn, "failed to deserialize", error) + } + Err(ProgramTransformerError::ParsingError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::ParsingError, + ); + log_or_bail!(warn, "failed to parse", error) + } + Err(ProgramTransformerError::DatabaseError(error)) => { + log_or_bail!(anyhow::bail, "database error for", error) + } + Err(ProgramTransformerError::AssetIndexError(error)) => { + log_or_bail!(anyhow::bail, "indexing error for ", error) + } + Err(error) => { + log_or_bail!(anyhow::bail, "failed to handle", error) + } + } + + msg.ack() + } + }); }; redis_messages.shutdown(); + while let Some(result) = pt_tasks.join_next().await { + result??; + } if !redis_tasks_fut.is_terminated() { redis_tasks_fut.await?; } @@ -84,8 +207,14 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { result } -async fn run_program_transformers() -> anyhow::Result<()> { - // let pt_accounts = ProgramTransformer::new() - - todo!() +fn create_notifier() -> DownloadMetadataNotifier { + Box::new( + move |_info: DownloadMetadataInfo| -> BoxFuture< + 'static, + Result<(), Box>, + > { + // TODO + Box::pin(async move { Ok(()) }) + }, + ) } diff --git a/nft_ingester2/src/postgres.rs b/nft_ingester2/src/postgres.rs index 6f7bc3c0d..05e535b4d 100644 --- a/nft_ingester2/src/postgres.rs +++ b/nft_ingester2/src/postgres.rs @@ -7,22 +7,20 @@ use { postgres::{PgConnectOptions, PgPoolOptions}, PgPool, }, - std::sync::Arc, tokio::time::{sleep, Duration}, }; -pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result> { +pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result { let options: PgConnectOptions = config.url.parse()?; PgPoolOptions::new() .min_connections(config.min_connections.try_into()?) .max_connections(config.max_connections.try_into()?) .connect_with(options) .await - .map(Arc::new) .map_err(Into::into) } -pub async fn metrics_pgpool(pgpool: Arc) { +pub async fn metrics_pgpool(pgpool: PgPool) { loop { pgpool_connections_set(PgpoolConnectionsKind::Total, pgpool.size() as usize); pgpool_connections_set(PgpoolConnectionsKind::Idle, pgpool.num_idle()); diff --git a/nft_ingester2/src/prom.rs b/nft_ingester2/src/prom.rs index 1b048671e..884407760 100644 --- a/nft_ingester2/src/prom.rs +++ b/nft_ingester2/src/prom.rs @@ -5,7 +5,7 @@ use { service::{make_service_fn, service_fn}, Body, Request, Response, Server, StatusCode, }, - prometheus::{IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder}, + prometheus::{IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder}, std::{net::SocketAddr, sync::Once}, tracing::{error, info}, }; @@ -37,6 +37,15 @@ lazy_static::lazy_static! { Opts::new("pgpool_connections_total", "Total number of connections in Postgres Pool"), &["kind"] ).unwrap(); + + static ref PROGRAM_TRANSFORMER_TASKS_TOTAL: IntGauge = IntGauge::new( + "program_transformer_tasks_total", "Number of tasks spawned for program transform" + ).unwrap(); + + static ref PROGRAM_TRANSFORMER_TASK_STATUS: IntCounterVec = IntCounterVec::new( + Opts::new("program_transformer_task_status", "Status of processed messages"), + &["status"], + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -54,6 +63,8 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(REDIS_XADD_STATUS); register!(REDIS_XACK_TOTAL); register!(PGPOOL_CONNECTIONS_TOTAL); + register!(PROGRAM_TRANSFORMER_TASKS_TOTAL); + register!(PROGRAM_TRANSFORMER_TASK_STATUS); VERSION .with_label_values(&[ @@ -137,3 +148,26 @@ pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { }]) .set(size as i64) } + +pub fn program_transformer_tasks_total_set(size: usize) { + PROGRAM_TRANSFORMER_TASKS_TOTAL.set(size as i64) +} + +#[derive(Debug, Clone, Copy)] +pub enum ProgramTransformerTaskStatusKind { + Success, + NotImplemented, + DeserializationError, + ParsingError, +} + +pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKind) { + PROGRAM_TRANSFORMER_TASK_STATUS + .with_label_values(&[match kind { + ProgramTransformerTaskStatusKind::Success => "success", + ProgramTransformerTaskStatusKind::NotImplemented => "not_implemented", + ProgramTransformerTaskStatusKind::DeserializationError => "deserialization_error", + ProgramTransformerTaskStatusKind::ParsingError => "parsing_error", + }]) + .inc() +} diff --git a/nft_ingester2/src/redis.rs b/nft_ingester2/src/redis.rs index 554f02c6e..b823f4fee 100644 --- a/nft_ingester2/src/redis.rs +++ b/nft_ingester2/src/redis.rs @@ -3,7 +3,8 @@ use { config::{ConfigIngesterRedis, ConfigIngesterRedisStreamType}, prom::{redis_xack_inc, redis_xlen_set}, }, - futures::future::{BoxFuture, FutureExt}, + futures::future::{BoxFuture, Fuse, FutureExt}, + program_transformers::{AccountInfo, TransactionInfo}, redis::{ aio::MultiplexedConnection, streams::{ @@ -12,6 +13,7 @@ use { }, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, + solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{ collections::HashMap, convert::Infallible, @@ -26,6 +28,9 @@ use { time::{sleep, Duration, Instant}, }, yellowstone_grpc_proto::{ + convert_from::{ + create_message_instructions, create_meta_inner_instructions, create_pubkey_vec, + }, prelude::{SubscribeUpdateAccount, SubscribeUpdateTransaction}, prost::Message, }, @@ -90,17 +95,16 @@ struct RedisStreamInfo { xack_max_in_process: usize, } -#[allow(clippy::large_enum_variant)] #[derive(Debug)] -pub enum RedisStreamMessage { - Account(SubscribeUpdateAccount), - Transaction(SubscribeUpdateTransaction), +pub enum ProgramTransformerInfo { + Account(AccountInfo), + Transaction(TransactionInfo), } #[derive(Debug)] pub struct RedisStreamMessageInfo { id: String, - data: RedisStreamMessage, + data: ProgramTransformerInfo, ack_tx: mpsc::UnboundedSender, } @@ -110,14 +114,69 @@ impl RedisStreamMessageInfo { StreamId { id, map }: StreamId, ack_tx: mpsc::UnboundedSender, ) -> anyhow::Result { + let to_anyhow = |error: String| anyhow::anyhow!(error); + let data = match map.get(&stream.stream_data_key) { Some(RedisValue::Data(vec)) => match stream.stream_type { ConfigIngesterRedisStreamType::Account => { - RedisStreamMessage::Account(SubscribeUpdateAccount::decode(vec.as_ref())?) + let SubscribeUpdateAccount { account, slot, .. } = + Message::decode(vec.as_ref())?; + + let account = account.ok_or_else(|| { + anyhow::anyhow!("received invalid SubscribeUpdateAccount") + })?; + + ProgramTransformerInfo::Account(AccountInfo { + slot, + pubkey: Pubkey::try_from(account.pubkey.as_slice())?, + owner: Pubkey::try_from(account.owner.as_slice())?, + data: account.data, + }) + } + ConfigIngesterRedisStreamType::Transaction => { + let SubscribeUpdateTransaction { transaction, slot } = + Message::decode(vec.as_ref())?; + + let transaction = transaction.ok_or_else(|| { + anyhow::anyhow!("received invalid SubscribeUpdateTransaction") + })?; + let tx = transaction.transaction.ok_or_else(|| { + anyhow::anyhow!( + "received invalid transaction in SubscribeUpdateTransaction" + ) + })?; + let message = tx.message.ok_or_else(|| { + anyhow::anyhow!("received invalid message in SubscribeUpdateTransaction") + })?; + let meta = transaction.meta.ok_or_else(|| { + anyhow::anyhow!("received invalid meta in SubscribeUpdateTransaction") + })?; + + let mut account_keys = + create_pubkey_vec(message.account_keys).map_err(to_anyhow)?; + for pubkey in + create_pubkey_vec(meta.loaded_writable_addresses).map_err(to_anyhow)? + { + account_keys.push(pubkey); + } + for pubkey in + create_pubkey_vec(meta.loaded_readonly_addresses).map_err(to_anyhow)? + { + account_keys.push(pubkey); + } + + ProgramTransformerInfo::Transaction(TransactionInfo { + slot, + signature: Signature::try_from(transaction.signature.as_slice())?, + account_keys, + message_instructions: create_message_instructions(message.instructions) + .map_err(to_anyhow)?, + meta_inner_instructions: create_meta_inner_instructions( + meta.inner_instructions, + ) + .map_err(to_anyhow)?, + }) } - ConfigIngesterRedisStreamType::Transaction => RedisStreamMessage::Transaction( - SubscribeUpdateTransaction::decode(vec.as_ref())?, - ), }, Some(_) => anyhow::bail!( "invalid data (key: {:?}) from stream {:?}", @@ -133,6 +192,10 @@ impl RedisStreamMessageInfo { Ok(Self { id, data, ack_tx }) } + pub const fn get_data(&self) -> &ProgramTransformerInfo { + &self.data + } + pub fn ack(self) -> anyhow::Result<()> { self.ack_tx .send(self.id) @@ -150,7 +213,7 @@ impl RedisStream { pub async fn new( config: ConfigIngesterRedis, mut connection: MultiplexedConnection, - ) -> anyhow::Result<(Self, BoxFuture<'static, anyhow::Result<()>>)> { + ) -> anyhow::Result<(Self, Fuse>>)> { // create group with consumer per stream for stream in config.streams.iter() { xgroup_create( @@ -209,15 +272,14 @@ impl RedisStream { task.await??; } Ok::<(), anyhow::Error>(()) - } - .boxed(); + }; Ok(( Self { shutdown, messages_rx, }, - spawned_tasks, + spawned_tasks.boxed().fuse(), )) } From 53fa481445b2b2c33d8bc712b25efb3144298863 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 29 Nov 2023 22:10:42 -0500 Subject: [PATCH 17/72] add force shutdown with signals --- Cargo.lock | 1 + Cargo.toml | 1 + nft_ingester2/Cargo.toml | 1 + nft_ingester2/src/grpc.rs | 27 +++++++++++----------- nft_ingester2/src/ingester.rs | 43 ++++++++++++++++++++--------------- nft_ingester2/src/redis.rs | 14 +++++++----- nft_ingester2/src/util.rs | 15 +++++++----- 7 files changed, 59 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd708440f..3f884075b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3727,6 +3727,7 @@ name = "nft_ingester2" version = "0.7.2" dependencies = [ "anyhow", + "async-stream", "atty", "cargo-lock", "clap 4.5.4", diff --git a/Cargo.toml b/Cargo.toml index 3cf0fcbfb..a666f375f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ anchor-client = "0.29.0" anchor-lang = "0.29.0" anyhow = "1.0.75" async-std = "1.0.0" +async-stream = "0.3.5" async-trait = "0.1.60" blockbuster = "2.2.1" atty = "0.2.14" diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml index 455f2902d..4ab4f7f80 100644 --- a/nft_ingester2/Cargo.toml +++ b/nft_ingester2/Cargo.toml @@ -7,6 +7,7 @@ publish = { workspace = true } [dependencies] anyhow = { workspace = true } +async-stream = { workspace = true } atty = { workspace = true } sqlx = { workspace = true, features = [ "macros", diff --git a/nft_ingester2/src/grpc.rs b/nft_ingester2/src/grpc.rs index aebb88b8a..01387c025 100644 --- a/nft_ingester2/src/grpc.rs +++ b/nft_ingester2/src/grpc.rs @@ -7,10 +7,10 @@ use { redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, std::{sync::Arc, time::Duration}, tokio::{ - signal::unix::SignalKind, task::JoinSet, time::{sleep, Instant}, }, + tracing::warn, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{prelude::subscribe_update::UpdateOneof, prost::Message}, }; @@ -59,15 +59,8 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { Ok(Err(error)) => break Err(error), Err(error) => break Err(error.into()), }, - signal = &mut shutdown => { - let signal = if signal == SignalKind::interrupt() { - "SIGINT" - } else if signal == SignalKind::terminate() { - "SIGTERM" - } else { - "UNKNOWN" - }; - tracing::warn!("{signal} received, waiting spawned tasks..."); + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); break Ok(()); }, msg = stream.next() => { @@ -138,9 +131,17 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { } }; - while let Some(result) = tasks.join_next().await { - result??; - } + tokio::select! { + Some(signal) = shutdown.next() => { + anyhow::bail!("{signal} received, force shutdown..."); + } + result = async move { + while let Some(result) = tasks.join_next().await { + result??; + } + Ok::<(), anyhow::Error>(()) + } => result?, + }; result } diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 0d7d43478..479e5bddb 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -9,7 +9,10 @@ use { redis::{metrics_xlen, ProgramTransformerInfo, RedisStream}, util::create_shutdown, }, - futures::future::{pending, BoxFuture, FusedFuture, FutureExt}, + futures::{ + future::{pending, BoxFuture, FusedFuture, FutureExt}, + stream::StreamExt, + }, program_transformers::{ error::ProgramTransformerError, DownloadMetadataInfo, DownloadMetadataNotifier, ProgramTransformer, @@ -19,7 +22,6 @@ use { Arc, }, tokio::{ - signal::unix::SignalKind, task::JoinSet, time::{sleep, Duration}, }, @@ -103,14 +105,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { Ok(Err(error)) => break Err(error), Err(error) => break Err(error.into()), }, - signal = &mut shutdown => { - let signal = if signal == SignalKind::interrupt() { - "SIGINT" - } else if signal == SignalKind::terminate() { - "SIGTERM" - } else { - "UNKNOWN" - }; + Some(signal) = shutdown.next() => { warn!("{signal} received, waiting spawned tasks..."); break Ok(()); }, @@ -195,14 +190,26 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { }); }; - redis_messages.shutdown(); - while let Some(result) = pt_tasks.join_next().await { - result??; - } - if !redis_tasks_fut.is_terminated() { - redis_tasks_fut.await?; - } - pgpool.close().await; + tokio::select! { + Some(signal) = shutdown.next() => { + anyhow::bail!("{signal} received, force shutdown..."); + } + result = async move { + // shutdown `prefetch` channel (but not Receiver) + redis_messages.shutdown(); + // wait all `program_transformer` spawned tasks + while let Some(result) = pt_tasks.join_next().await { + result??; + } + // wait all `ack` spawned tasks + if !redis_tasks_fut.is_terminated() { + redis_tasks_fut.await?; + } + // shutdown database connection + pgpool.close().await; + Ok::<(), anyhow::Error>(()) + } => result?, + }; result } diff --git a/nft_ingester2/src/redis.rs b/nft_ingester2/src/redis.rs index b823f4fee..1baec8799 100644 --- a/nft_ingester2/src/redis.rs +++ b/nft_ingester2/src/redis.rs @@ -251,7 +251,7 @@ impl RedisStream { .collect::>(); // spawn xack tasks - let mut tasks = ack_tasks + let ack_jh_vec = ack_tasks .into_iter() .map(|(stream, ack_rx)| { let connection = connection.clone(); @@ -261,15 +261,16 @@ impl RedisStream { // spawn prefetch task let (messages_tx, messages_rx) = mpsc::channel(config.prefetch_queue_size); - tasks.push(tokio::spawn({ + let jh_prefetch = tokio::spawn({ let shutdown = Arc::clone(&shutdown); async move { Self::run_prefetch(config, streams, connection, messages_tx, shutdown).await } - })); + }); // merge spawned xack / prefetch tasks let spawned_tasks = async move { - for task in tasks.into_iter() { - task.await??; + jh_prefetch.await??; + for jh in ack_jh_vec.into_iter() { + jh.await??; } Ok::<(), anyhow::Error>(()) }; @@ -287,8 +288,9 @@ impl RedisStream { self.messages_rx.recv().await } - pub fn shutdown(self) { + pub fn shutdown(mut self) { self.shutdown.store(true, Ordering::Relaxed); + tokio::spawn(async move { while self.messages_rx.recv().await.is_some() {} }); } async fn run_prefetch( diff --git a/nft_ingester2/src/util.rs b/nft_ingester2/src/util.rs index 6de97e42f..0a7800a12 100644 --- a/nft_ingester2/src/util.rs +++ b/nft_ingester2/src/util.rs @@ -1,15 +1,18 @@ use { - futures::future::{BoxFuture, FutureExt}, + async_stream::stream, + futures::stream::{BoxStream, StreamExt}, tokio::signal::unix::{signal, SignalKind}, }; -pub fn create_shutdown() -> anyhow::Result> { +pub fn create_shutdown() -> anyhow::Result> { let mut sigint = signal(SignalKind::interrupt())?; let mut sigterm = signal(SignalKind::terminate())?; - Ok(async move { - tokio::select! { - _ = sigint.recv() => SignalKind::interrupt(), - _ = sigterm.recv() => SignalKind::terminate(), + Ok(stream! { + loop { + yield tokio::select! { + _ = sigint.recv() => "SIGINT", + _ = sigterm.recv() => "SIGTERM", + }; } } .boxed()) From 11b4151d3e7492b638f9a865e3a1645d24461279 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 30 Nov 2023 21:20:38 -0500 Subject: [PATCH 18/72] insert download metadata tasks --- Cargo.lock | 14 +++ metaplex-rpc-proxy/Cargo.toml | 2 + nft_ingester/src/transaction_notifications.rs | 2 +- nft_ingester2/Cargo.toml | 5 ++ nft_ingester2/config-run.yml | 6 +- nft_ingester2/src/config.rs | 29 ++++++- nft_ingester2/src/ingester.rs | 85 ++++++++++++++----- nft_ingester2/src/main.rs | 1 + nft_ingester2/src/prom.rs | 11 ++- program_transformers/Cargo.toml | 1 + program_transformers/src/lib.rs | 3 +- 11 files changed, 132 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f884075b..a8f8fa573 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3537,6 +3537,8 @@ dependencies = [ "log", "proxy-wasm", "regex", + "wasi 0.7.0", + "wasm-bindgen", ] [[package]] @@ -3730,7 +3732,9 @@ dependencies = [ "async-stream", "atty", "cargo-lock", + "chrono", "clap 4.5.4", + "digital_asset_types", "futures", "git-version", "hyper", @@ -3742,7 +3746,10 @@ dependencies = [ "program_transformers", "prometheus", "redis 0.25.3", + "rust-crypto", + "sea-orm", "serde", + "serde_json", "serde_yaml", "solana-sdk", "sqlx", @@ -4633,6 +4640,7 @@ dependencies = [ "mpl-bubblegum", "num-traits", "sea-orm", + "serde", "serde_json", "solana-sdk", "solana-transaction-status", @@ -8166,6 +8174,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" diff --git a/metaplex-rpc-proxy/Cargo.toml b/metaplex-rpc-proxy/Cargo.toml index 06b801543..2e69785dd 100644 --- a/metaplex-rpc-proxy/Cargo.toml +++ b/metaplex-rpc-proxy/Cargo.toml @@ -13,6 +13,8 @@ lazy_static = { workspace = true } log = { workspace = true } proxy-wasm = { workspace = true } regex = { workspace = true } +wasi = { workspace = true } +wasm-bindgen = { workspace = true } [lints] workspace = true diff --git a/nft_ingester/src/transaction_notifications.rs b/nft_ingester/src/transaction_notifications.rs index 5335de6dc..df005bc56 100644 --- a/nft_ingester/src/transaction_notifications.rs +++ b/nft_ingester/src/transaction_notifications.rs @@ -10,7 +10,7 @@ use { log::{debug, error}, plerkle_messenger::{ConsumptionType, Messenger, MessengerConfig, RecvData}, plerkle_serialization::root_as_transaction_info, - program_transformers::{error::ProgramTransformerResult, ProgramTransformer, TransactionInfo}, + program_transformers::ProgramTransformer, sqlx::{Pool, Postgres}, std::sync::Arc, tokio::{ diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml index 4ab4f7f80..bf3f590ec 100644 --- a/nft_ingester2/Cargo.toml +++ b/nft_ingester2/Cargo.toml @@ -17,7 +17,9 @@ sqlx = { workspace = true, features = [ "offline", "json", ] } +chrono = { workspace = true } clap = { workspace = true, features = ["cargo", "derive"] } +digital_asset_types = { workspace = true } futures = { workspace = true } hyper = { workspace = true, features = ["server"] } json5 = { workspace = true } @@ -28,7 +30,10 @@ opentelemetry_sdk = { workspace = true, features = ["trace"] } program_transformers = { workspace = true } prometheus = { workspace = true } redis = { workspace = true, features = ["tokio-comp", "tokio-native-tls-comp"] } +rust-crypto = { workspace = true } +sea-orm = { workspace = true, features = ["sqlx-postgres"] } serde = { workspace = true } +serde_json = { workspace = true } serde_yaml = { workspace = true } solana-sdk = { workspace = true } # only prom rn tokio = { workspace = true, features = [ diff --git a/nft_ingester2/config-run.yml b/nft_ingester2/config-run.yml index 19a1b3f84..3af3ad2e7 100644 --- a/nft_ingester2/config-run.yml +++ b/nft_ingester2/config-run.yml @@ -20,7 +20,9 @@ redis: postgres: url: postgres://solana:solana@localhost/solana min_connections: 10 - max_connections: 25 + max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible program_transformer: transactions_cl_audits: false - max_tasks_in_process: 100 + max_tasks_in_process: 40 +download_metadata_handler: + max_attempts: 3 diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs index 003985f27..d01bfeeaa 100644 --- a/nft_ingester2/src/config.rs +++ b/nft_ingester2/src/config.rs @@ -3,6 +3,7 @@ use { serde::{de, Deserialize}, std::{net::SocketAddr, path::Path, time::Duration}, tokio::fs, + tracing::warn, yellowstone_grpc_tools::config::{ deserialize_usize_str, ConfigGrpcRequestAccounts, ConfigGrpcRequestCommitment, ConfigGrpcRequestTransactions, @@ -154,6 +155,15 @@ pub struct ConfigIngester { pub redis: ConfigIngesterRedis, pub postgres: ConfigIngesterPostgres, pub program_transformer: ConfigIngesterProgramTransformer, + pub download_metadata_handler: ConfigDownloadMetadataHandler, +} + +impl ConfigIngester { + pub fn check(&self) { + if self.postgres.max_connections < self.program_transformer.max_tasks_in_process { + warn!("`postgres.max_connections` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible"); + } + } } #[derive(Debug, Deserialize)] @@ -305,7 +315,7 @@ impl ConfigIngesterPostgres { } pub const fn default_max_connections() -> usize { - 25 + 50 } } @@ -326,6 +336,21 @@ impl ConfigIngesterProgramTransformer { } pub const fn default_max_tasks_in_process() -> usize { - 100 + 40 + } +} + +#[derive(Debug, Clone, Copy, Deserialize)] +pub struct ConfigDownloadMetadataHandler { + #[serde( + default = "ConfigDownloadMetadataHandler::default_max_attempts", + deserialize_with = "deserialize_usize_str" + )] + pub max_attempts: usize, +} + +impl ConfigDownloadMetadataHandler { + pub const fn default_max_attempts() -> usize { + 3 } } diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 479e5bddb..22f4311c2 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -1,14 +1,17 @@ use { crate::{ - config::ConfigIngester, + config::{ConfigDownloadMetadataHandler, ConfigIngester}, postgres::{create_pool as pg_create_pool, metrics_pgpool}, prom::{ - program_transformer_task_status_inc, program_transformer_tasks_total_set, - ProgramTransformerTaskStatusKind, + download_metadata_inserted_total_inc, program_transformer_task_status_inc, + program_transformer_tasks_total_set, ProgramTransformerTaskStatusKind, }, redis::{metrics_xlen, ProgramTransformerInfo, RedisStream}, util::create_shutdown, }, + chrono::Utc, + crypto::{digest::Digest, sha2::Sha256}, + digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, futures::{ future::{pending, BoxFuture, FusedFuture, FutureExt}, stream::StreamExt, @@ -17,9 +20,18 @@ use { error::ProgramTransformerError, DownloadMetadataInfo, DownloadMetadataNotifier, ProgramTransformer, }, - std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, + sea_orm::{ + entity::{ActiveModelTrait, ActiveValue}, + error::{DbErr, RuntimeErr}, + SqlxPostgresConnector, + }, + sqlx::{Error as SqlxError, PgPool}, + std::{ + borrow::Cow, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }, tokio::{ task::JoinSet, @@ -29,8 +41,6 @@ use { }; pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { - println!("{:#?}", config); - // connect to Redis let client = redis::Client::open(config.redis.url.clone())?; let connection = client.get_multiplexed_tokio_connection().await?; @@ -62,12 +72,12 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // program transforms related let pt_accounts = Arc::new(ProgramTransformer::new( pgpool.clone(), - create_notifier(), + create_download_metadata_notifier(pgpool.clone(), config.download_metadata_handler)?, false, )); let pt_transactions = Arc::new(ProgramTransformer::new( pgpool.clone(), - create_notifier(), + create_download_metadata_notifier(pgpool.clone(), config.download_metadata_handler)?, config.program_transformer.transactions_cl_audits, )); let pt_max_tasks_in_process = config.program_transformer.max_tasks_in_process; @@ -214,14 +224,49 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { result } -fn create_notifier() -> DownloadMetadataNotifier { - Box::new( - move |_info: DownloadMetadataInfo| -> BoxFuture< - 'static, - Result<(), Box>, - > { - // TODO - Box::pin(async move { Ok(()) }) - }, - ) +fn create_download_metadata_notifier( + pgpool: PgPool, + config: ConfigDownloadMetadataHandler, +) -> anyhow::Result { + let max_attempts = config.max_attempts.try_into()?; + Ok(Box::new(move |info: DownloadMetadataInfo| -> BoxFuture< + 'static, + Result<(), Box>, + > { + let pgpool = pgpool.clone(); + Box::pin(async move { + const NAME: &str = "DownloadMetadata"; + + let data = serde_json::to_value(info)?; + + let mut hasher = Sha256::new(); + hasher.input(NAME.as_bytes()); + hasher.input(serde_json::to_vec(&data)?.as_slice()); + let hash = hasher.result_str(); + + let model = tasks::ActiveModel { + id: ActiveValue::Set(hash), + task_type: ActiveValue::Set(NAME.to_owned()), + data: ActiveValue::Set(data), + status: ActiveValue::Set(TaskStatus::Pending), + created_at: ActiveValue::Set(Utc::now().naive_utc()), + locked_until: ActiveValue::Set(None), + locked_by: ActiveValue::Set(None), + max_attempts: ActiveValue::Set(max_attempts), + attempts: ActiveValue::Set(0), + duration: ActiveValue::Set(None), + errors: ActiveValue::Set(None), + }; + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pgpool); + + match model.insert(&conn).await.map(|_mode| ()) { + // skip unique_violation error + Err(DbErr::Query(RuntimeErr::SqlxError(SqlxError::Database(dberr)))) if dberr.code() == Some(Cow::Borrowed("23505")) => {}, + value => value?, + }; + download_metadata_inserted_total_inc(); + + Ok(()) + }) + })) } diff --git a/nft_ingester2/src/main.rs b/nft_ingester2/src/main.rs index 0bd1f9b8d..7622b8aec 100644 --- a/nft_ingester2/src/main.rs +++ b/nft_ingester2/src/main.rs @@ -70,6 +70,7 @@ async fn main() -> anyhow::Result<()> { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; + config.check(); ingester::run(config).await } } diff --git a/nft_ingester2/src/prom.rs b/nft_ingester2/src/prom.rs index 884407760..e81de4255 100644 --- a/nft_ingester2/src/prom.rs +++ b/nft_ingester2/src/prom.rs @@ -5,7 +5,7 @@ use { service::{make_service_fn, service_fn}, Body, Request, Response, Server, StatusCode, }, - prometheus::{IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder}, + prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder}, std::{net::SocketAddr, sync::Once}, tracing::{error, info}, }; @@ -46,6 +46,10 @@ lazy_static::lazy_static! { Opts::new("program_transformer_task_status", "Status of processed messages"), &["status"], ).unwrap(); + + static ref DOWNLOAD_METADATA_INSERTED_TOTAL: IntCounter = IntCounter::new( + "download_metadata_inserted_total", "Total number of inserted tasks for download metadata" + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -65,6 +69,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(PGPOOL_CONNECTIONS_TOTAL); register!(PROGRAM_TRANSFORMER_TASKS_TOTAL); register!(PROGRAM_TRANSFORMER_TASK_STATUS); + register!(DOWNLOAD_METADATA_INSERTED_TOTAL); VERSION .with_label_values(&[ @@ -171,3 +176,7 @@ pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKin }]) .inc() } + +pub fn download_metadata_inserted_total_inc() { + DOWNLOAD_METADATA_INSERTED_TOTAL.inc() +} diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index 08f784632..aa96212e7 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -19,6 +19,7 @@ heck = { workspace = true } mpl-bubblegum = { workspace = true } num-traits = { workspace = true } sea-orm = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index edde20ca3..db52e55d4 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -20,6 +20,7 @@ use { entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, }, + serde::Serialize, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, sqlx::PgPool, @@ -52,7 +53,7 @@ pub struct TransactionInfo { pub meta_inner_instructions: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DownloadMetadataInfo { asset_data_id: Vec, uri: String, From 8fca2c22bf27fd2449ddd76af790b22cfff2744c Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Fri, 1 Dec 2023 22:34:46 -0500 Subject: [PATCH 19/72] download-metadata subtask --- Cargo.lock | 1 + nft_ingester2/Cargo.toml | 1 + nft_ingester2/config-download-metadata.yml | 12 + .../{config-run.yml => config-ingester.yml} | 2 +- nft_ingester2/src/config.rs | 67 ++++- nft_ingester2/src/download_metadata.rs | 236 ++++++++++++++++++ nft_ingester2/src/ingester.rs | 25 +- nft_ingester2/src/main.rs | 19 +- nft_ingester2/src/tracing.rs | 3 +- program_transformers/src/lib.rs | 4 +- 10 files changed, 345 insertions(+), 25 deletions(-) create mode 100644 nft_ingester2/config-download-metadata.yml rename nft_ingester2/{config-run.yml => config-ingester.yml} (97%) create mode 100644 nft_ingester2/src/download_metadata.rs diff --git a/Cargo.lock b/Cargo.lock index a8f8fa573..347dbf5c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3746,6 +3746,7 @@ dependencies = [ "program_transformers", "prometheus", "redis 0.25.3", + "reqwest", "rust-crypto", "sea-orm", "serde", diff --git a/nft_ingester2/Cargo.toml b/nft_ingester2/Cargo.toml index bf3f590ec..51afb7b09 100644 --- a/nft_ingester2/Cargo.toml +++ b/nft_ingester2/Cargo.toml @@ -30,6 +30,7 @@ opentelemetry_sdk = { workspace = true, features = ["trace"] } program_transformers = { workspace = true } prometheus = { workspace = true } redis = { workspace = true, features = ["tokio-comp", "tokio-native-tls-comp"] } +reqwest = { workspace = true } rust-crypto = { workspace = true } sea-orm = { workspace = true, features = ["sqlx-postgres"] } serde = { workspace = true } diff --git a/nft_ingester2/config-download-metadata.yml b/nft_ingester2/config-download-metadata.yml new file mode 100644 index 000000000..c42307be4 --- /dev/null +++ b/nft_ingester2/config-download-metadata.yml @@ -0,0 +1,12 @@ +# Important: only ONE `download-metadata` instance is supported right now! +prometheus: 127.0.0.1:8875 +postgres: + url: postgres://solana:solana@localhost/solana + min_connections: 10 + max_connections: 50 +download_metadata: + max_in_process: 50 # maximum tasks in process (downloading metadata) + prefetch_queue_size: 100 + limit_to_fetch: 200 # maximum number of tasks fetched from database + wait_tasks_max_idle_ms: 100 # if we do not have pending tasks, wait max ms + download_timeout_ms: 5_000 diff --git a/nft_ingester2/config-run.yml b/nft_ingester2/config-ingester.yml similarity index 97% rename from nft_ingester2/config-run.yml rename to nft_ingester2/config-ingester.yml index 3af3ad2e7..bbb377a81 100644 --- a/nft_ingester2/config-run.yml +++ b/nft_ingester2/config-ingester.yml @@ -24,5 +24,5 @@ postgres: program_transformer: transactions_cl_audits: false max_tasks_in_process: 40 -download_metadata_handler: +download_metadata: max_attempts: 3 diff --git a/nft_ingester2/src/config.rs b/nft_ingester2/src/config.rs index d01bfeeaa..c3c45531e 100644 --- a/nft_ingester2/src/config.rs +++ b/nft_ingester2/src/config.rs @@ -155,7 +155,7 @@ pub struct ConfigIngester { pub redis: ConfigIngesterRedis, pub postgres: ConfigIngesterPostgres, pub program_transformer: ConfigIngesterProgramTransformer, - pub download_metadata_handler: ConfigDownloadMetadataHandler, + pub download_metadata: ConfigIngesterDownloadMetadata, } impl ConfigIngester { @@ -341,16 +341,75 @@ impl ConfigIngesterProgramTransformer { } #[derive(Debug, Clone, Copy, Deserialize)] -pub struct ConfigDownloadMetadataHandler { +pub struct ConfigIngesterDownloadMetadata { #[serde( - default = "ConfigDownloadMetadataHandler::default_max_attempts", + default = "ConfigIngesterDownloadMetadata::default_max_attempts", deserialize_with = "deserialize_usize_str" )] pub max_attempts: usize, } -impl ConfigDownloadMetadataHandler { +impl ConfigIngesterDownloadMetadata { pub const fn default_max_attempts() -> usize { 3 } } + +#[derive(Debug, Deserialize)] +pub struct ConfigDownloadMetadata { + pub postgres: ConfigIngesterPostgres, + pub download_metadata: ConfigDownloadMetadataOpts, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +pub struct ConfigDownloadMetadataOpts { + #[serde( + default = "ConfigDownloadMetadataOpts::default_max_in_process", + deserialize_with = "deserialize_usize_str" + )] + pub max_in_process: usize, + #[serde( + default = "ConfigDownloadMetadataOpts::default_prefetch_queue_size", + deserialize_with = "deserialize_usize_str" + )] + pub prefetch_queue_size: usize, + #[serde( + default = "ConfigDownloadMetadataOpts::default_limit_to_fetch", + deserialize_with = "deserialize_usize_str" + )] + pub limit_to_fetch: usize, + #[serde( + default = "ConfigDownloadMetadataOpts::default_wait_tasks_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "wait_tasks_max_idle_ms" + )] + pub wait_tasks_max_idle: Duration, + #[serde( + default = "ConfigDownloadMetadataOpts::default_download_timeout", + deserialize_with = "deserialize_duration_str", + rename = "download_timeout_ms" + )] + pub download_timeout: Duration, +} + +impl ConfigDownloadMetadataOpts { + pub const fn default_max_in_process() -> usize { + 50 + } + + pub const fn default_prefetch_queue_size() -> usize { + 100 + } + + pub const fn default_limit_to_fetch() -> usize { + 200 + } + + pub const fn default_wait_tasks_max_idle() -> Duration { + Duration::from_millis(100) + } + + pub const fn default_download_timeout() -> Duration { + Duration::from_millis(5_000) + } +} diff --git a/nft_ingester2/src/download_metadata.rs b/nft_ingester2/src/download_metadata.rs new file mode 100644 index 000000000..4392710b2 --- /dev/null +++ b/nft_ingester2/src/download_metadata.rs @@ -0,0 +1,236 @@ +use { + crate::{ + config::{ConfigDownloadMetadata, ConfigDownloadMetadataOpts}, + postgres::{create_pool as pg_create_pool, metrics_pgpool}, + util::create_shutdown, + }, + digital_asset_types::dao::{asset_data, sea_orm_active_enums::TaskStatus, tasks}, + futures::{ + future::{pending, FutureExt}, + stream::StreamExt, + }, + program_transformers::DownloadMetadataInfo, + reqwest::{ClientBuilder, StatusCode}, + sea_orm::{ + entity::{ActiveValue, ColumnTrait, EntityTrait}, + query::{Condition, Order, QueryFilter, QueryOrder, QuerySelect}, + sea_query::expr::Expr, + SqlxPostgresConnector, TransactionTrait, + }, + sqlx::PgPool, + std::{sync::Arc, time::Duration}, + tokio::{ + sync::{mpsc, Notify}, + task::JoinSet, + time::sleep, + }, + tracing::{info, warn}, +}; + +pub const TASK_TYPE: &str = "DownloadMetadata"; + +pub async fn run(config: ConfigDownloadMetadata) -> anyhow::Result<()> { + let mut shutdown = create_shutdown()?; + + // open connection to postgres + let pool = pg_create_pool(config.postgres).await?; + tokio::spawn({ + let pool = pool.clone(); + async move { metrics_pgpool(pool).await } + }); + + // reset previously runned tasks + tokio::select! { + result = reset_pending_tasks(pool.clone()) => { + let updated = result?; + info!("Reset {updated} tasks to Pending status"); + }, + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); + return Ok(()) + }, + } + + // prefetch queue + let (tasks_tx, mut tasks_rx) = mpsc::channel(config.download_metadata.prefetch_queue_size); + let prefetch_shutdown = Arc::new(Notify::new()); + let prefetch_jh = { + let pool = pool.clone(); + let download_metadata = config.download_metadata; + let shutdown = Arc::clone(&prefetch_shutdown); + async move { + tokio::select! { + result = get_pending_tasks(pool, tasks_tx, download_metadata) => result, + _ = shutdown.notified() => Ok(()) + } + } + }; + tokio::pin!(prefetch_jh); + + // process tasks + let mut tasks = JoinSet::new(); + loop { + let pending_task_fut = if tasks.len() >= config.download_metadata.max_in_process { + pending().boxed() + } else { + tasks_rx.recv().boxed() + }; + + let tasks_fut = if tasks.is_empty() { + pending().boxed() + } else { + tasks.join_next().boxed() + }; + + tokio::select! { + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); + break Ok(()); + }, + result = &mut prefetch_jh => break result, + Some(result) = tasks_fut => { + result??; + }, + Some(pending_task) = pending_task_fut => { + tasks.spawn(execute_task(pool.clone(), pending_task, config.download_metadata.download_timeout)); + } + }; + }?; + + tokio::select! { + Some(signal) = shutdown.next() => { + anyhow::bail!("{signal} received, force shutdown..."); + } + result = async move { + // shutdown `prefetch` channel + prefetch_shutdown.notify_one(); + // wait all spawned tasks + while let Some(result) = tasks.join_next().await { + result??; + } + // shutdown database connection + pool.close().await; + Ok::<(), anyhow::Error>(()) + } => result, + } +} + +// On startup reset tasks status +async fn reset_pending_tasks(pool: PgPool) -> anyhow::Result { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + tasks::Entity::update_many() + .set(tasks::ActiveModel { + status: ActiveValue::Set(TaskStatus::Pending), + ..Default::default() + }) + .filter( + Condition::all() + .add(tasks::Column::Status.eq(TaskStatus::Running)) + .add(tasks::Column::TaskType.eq(TASK_TYPE)), + ) + .exec(&conn) + .await + .map(|result| result.rows_affected) + .map_err(Into::into) +} + +// Select Pending tasks, update status to Running and send to prefetch queue +async fn get_pending_tasks( + pool: PgPool, + tasks_tx: mpsc::Sender, + config: ConfigDownloadMetadataOpts, +) -> anyhow::Result<()> { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + loop { + let pending_tasks = tasks::Entity::find() + .filter( + Condition::all() + .add(tasks::Column::Status.eq(TaskStatus::Pending)) + .add( + Expr::col(tasks::Column::Attempts) + .less_or_equal(Expr::col(tasks::Column::MaxAttempts)), + ), + ) + .order_by(tasks::Column::Attempts, Order::Asc) + .order_by(tasks::Column::CreatedAt, Order::Desc) + .limit(config.limit_to_fetch as u64) + .all(&conn) + .await?; + + if pending_tasks.is_empty() { + sleep(config.wait_tasks_max_idle).await; + } else { + tasks::Entity::update_many() + .set(tasks::ActiveModel { + status: ActiveValue::Set(TaskStatus::Running), + ..Default::default() + }) + .filter(tasks::Column::Id.is_in(pending_tasks.iter().map(|v| v.id.clone()))) + .exec(&conn) + .await?; + + for task in pending_tasks { + tasks_tx + .send(task) + .await + .map_err(|_error| anyhow::anyhow!("failed to send task to prefetch queue"))?; + } + } + } +} + +// Try to download metadata and remove task with asset_data update or update tasks to Pending/Failed +async fn execute_task(pool: PgPool, task: tasks::Model, timeout: Duration) -> anyhow::Result<()> { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + match download_metadata(task.data, timeout).await { + Ok((asset_data_id, metadata)) => { + // Remove task and set metadata in transacstion + let txn = conn.begin().await?; + tasks::Entity::delete_by_id(task.id).exec(&txn).await?; + asset_data::Entity::update(asset_data::ActiveModel { + id: ActiveValue::Unchanged(asset_data_id), + metadata: ActiveValue::Set(metadata), + reindex: ActiveValue::Set(Some(false)), + ..Default::default() + }) + .exec(&txn) + .await?; + txn.commit().await?; + } + Err(error) => { + let status = if task.attempts + 1 == task.max_attempts { + TaskStatus::Failed + } else { + TaskStatus::Pending + }; + tasks::Entity::update(tasks::ActiveModel { + id: ActiveValue::Unchanged(task.id), + status: ActiveValue::Set(status), + attempts: ActiveValue::Set(task.attempts + 1), + errors: ActiveValue::Set(Some(error.to_string())), + ..Default::default() + }) + .exec(&conn) + .await?; + } + } + Ok(()) +} + +async fn download_metadata( + data: serde_json::Value, + timeout: Duration, +) -> anyhow::Result<(Vec, serde_json::Value)> { + let (id, uri) = serde_json::from_value::(data)?.into_inner(); + + // Need to check for malicious sites ? + let client = ClientBuilder::new().timeout(timeout).build()?; + let response = client.get(uri).send().await?; + + anyhow::ensure!( + response.status() == StatusCode::OK, + "HttpError status_code: {}", + response.status().as_str() + ); + Ok((id, response.json().await?)) +} diff --git a/nft_ingester2/src/ingester.rs b/nft_ingester2/src/ingester.rs index 22f4311c2..82cdc3cb5 100644 --- a/nft_ingester2/src/ingester.rs +++ b/nft_ingester2/src/ingester.rs @@ -1,6 +1,7 @@ use { crate::{ - config::{ConfigDownloadMetadataHandler, ConfigIngester}, + config::{ConfigIngester, ConfigIngesterDownloadMetadata}, + download_metadata::TASK_TYPE, postgres::{create_pool as pg_create_pool, metrics_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, @@ -72,12 +73,12 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // program transforms related let pt_accounts = Arc::new(ProgramTransformer::new( pgpool.clone(), - create_download_metadata_notifier(pgpool.clone(), config.download_metadata_handler)?, + create_download_metadata_notifier(pgpool.clone(), config.download_metadata)?, false, )); let pt_transactions = Arc::new(ProgramTransformer::new( pgpool.clone(), - create_download_metadata_notifier(pgpool.clone(), config.download_metadata_handler)?, + create_download_metadata_notifier(pgpool.clone(), config.download_metadata)?, config.program_transformer.transactions_cl_audits, )); let pt_max_tasks_in_process = config.program_transformer.max_tasks_in_process; @@ -95,7 +96,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // read and process messages in the loop let mut shutdown = create_shutdown()?; - let result = loop { + loop { pt_tasks_len.store(pt_tasks.len(), Ordering::Relaxed); let redis_messages_recv = if pt_tasks.len() == pt_max_tasks_in_process { @@ -198,7 +199,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { msg.ack() } }); - }; + }?; tokio::select! { Some(signal) = shutdown.next() => { @@ -218,15 +219,13 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // shutdown database connection pgpool.close().await; Ok::<(), anyhow::Error>(()) - } => result?, - }; - - result + } => result, + } } fn create_download_metadata_notifier( pgpool: PgPool, - config: ConfigDownloadMetadataHandler, + config: ConfigIngesterDownloadMetadata, ) -> anyhow::Result { let max_attempts = config.max_attempts.try_into()?; Ok(Box::new(move |info: DownloadMetadataInfo| -> BoxFuture< @@ -235,18 +234,16 @@ fn create_download_metadata_notifier( > { let pgpool = pgpool.clone(); Box::pin(async move { - const NAME: &str = "DownloadMetadata"; - let data = serde_json::to_value(info)?; let mut hasher = Sha256::new(); - hasher.input(NAME.as_bytes()); + hasher.input(TASK_TYPE.as_bytes()); hasher.input(serde_json::to_vec(&data)?.as_slice()); let hash = hasher.result_str(); let model = tasks::ActiveModel { id: ActiveValue::Set(hash), - task_type: ActiveValue::Set(NAME.to_owned()), + task_type: ActiveValue::Set(TASK_TYPE.to_owned()), data: ActiveValue::Set(data), status: ActiveValue::Set(TaskStatus::Pending), created_at: ActiveValue::Set(Utc::now().naive_utc()), diff --git a/nft_ingester2/src/main.rs b/nft_ingester2/src/main.rs index 7622b8aec..9ca97d111 100644 --- a/nft_ingester2/src/main.rs +++ b/nft_ingester2/src/main.rs @@ -1,6 +1,9 @@ use { crate::{ - config::{load as config_load, ConfigGrpc, ConfigIngester, ConfigPrometheus}, + config::{ + load as config_load, ConfigDownloadMetadata, ConfigGrpc, ConfigIngester, + ConfigPrometheus, + }, prom::run_server as prometheus_run_server, tracing::init as tracing_init, }, @@ -10,6 +13,7 @@ use { }; mod config; +mod download_metadata; mod grpc; mod ingester; mod postgres; @@ -40,8 +44,11 @@ enum ArgsAction { #[command(name = "grpc2redis")] Grpc, /// Run ingester process (process events from Redis) - #[command(name = "run")] + #[command(name = "ingester")] Ingester, + /// Run metadata downloader + #[command(name = "download-metadata")] + DownloadMetadata, } #[tokio::main] @@ -58,7 +65,7 @@ async fn main() -> anyhow::Result<()> { prometheus_run_server(address)?; } - // Run grpc / ingester + // Run grpc / ingester / download-metadata match args.action { ArgsAction::Grpc => { let config = config_load::(&args.config) @@ -73,5 +80,11 @@ async fn main() -> anyhow::Result<()> { config.check(); ingester::run(config).await } + ArgsAction::DownloadMetadata => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + download_metadata::run(config).await + } } } diff --git a/nft_ingester2/src/tracing.rs b/nft_ingester2/src/tracing.rs index 36ba4db56..2d50f785c 100644 --- a/nft_ingester2/src/tracing.rs +++ b/nft_ingester2/src/tracing.rs @@ -12,7 +12,8 @@ pub fn init() -> anyhow::Result<()> { .install_batch(opentelemetry_sdk::runtime::Tokio)?; let jeager_layer = tracing_opentelemetry::layer().with_tracer(open_tracer); - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let env_filter = EnvFilter::builder() + .parse(env::var(EnvFilter::DEFAULT_ENV).unwrap_or_else(|_| "info,sqlx=warn".to_owned()))?; let is_atty = atty::is(atty::Stream::Stdout) && atty::is(atty::Stream::Stderr); let io_layer = tracing_subscriber::fmt::layer().with_ansi(is_atty); diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index db52e55d4..6b6259d01 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -20,7 +20,7 @@ use { entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, }, - serde::Serialize, + serde::{Deserialize, Serialize}, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, sqlx::PgPool, @@ -53,7 +53,7 @@ pub struct TransactionInfo { pub meta_inner_instructions: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct DownloadMetadataInfo { asset_data_id: Vec, uri: String, From fcc18d694bb6ec442f3de9933e26236758f59d22 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 10 Apr 2024 19:32:36 +0200 Subject: [PATCH 20/72] refactor: rename nft_ingest2 --- Cargo.lock | 80 +++++++++---------- Cargo.toml | 2 +- {nft_ingester2 => grpc-ingest}/Cargo.toml | 2 +- {nft_ingester2 => grpc-ingest}/README.md | 0 {nft_ingester2 => grpc-ingest}/build.rs | 0 .../config-download-metadata.yml | 0 .../config-grpc2redis.yml | 0 .../config-ingester.yml | 0 {nft_ingester2 => grpc-ingest}/src/config.rs | 0 .../src/download_metadata.rs | 0 {nft_ingester2 => grpc-ingest}/src/grpc.rs | 0 .../src/ingester.rs | 0 {nft_ingester2 => grpc-ingest}/src/main.rs | 0 .../src/postgres.rs | 0 {nft_ingester2 => grpc-ingest}/src/prom.rs | 0 {nft_ingester2 => grpc-ingest}/src/redis.rs | 0 {nft_ingester2 => grpc-ingest}/src/tracing.rs | 0 {nft_ingester2 => grpc-ingest}/src/util.rs | 0 {nft_ingester2 => grpc-ingest}/src/version.rs | 0 19 files changed, 42 insertions(+), 42 deletions(-) rename {nft_ingester2 => grpc-ingest}/Cargo.toml (98%) rename {nft_ingester2 => grpc-ingest}/README.md (100%) rename {nft_ingester2 => grpc-ingest}/build.rs (100%) rename {nft_ingester2 => grpc-ingest}/config-download-metadata.yml (100%) rename {nft_ingester2 => grpc-ingest}/config-grpc2redis.yml (100%) rename {nft_ingester2 => grpc-ingest}/config-ingester.yml (100%) rename {nft_ingester2 => grpc-ingest}/src/config.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/download_metadata.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/grpc.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/ingester.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/main.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/postgres.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/prom.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/redis.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/tracing.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/util.rs (100%) rename {nft_ingester2 => grpc-ingest}/src/version.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 347dbf5c5..40e3b3e72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,46 @@ dependencies = [ "tokio", ] +[[package]] +name = "das-grpc-ingest" +version = "0.7.2" +dependencies = [ + "anyhow", + "async-stream", + "atty", + "cargo-lock", + "chrono", + "clap 4.5.4", + "digital_asset_types", + "futures", + "git-version", + "hyper", + "json5", + "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", + "opentelemetry_sdk", + "program_transformers", + "prometheus", + "redis 0.25.3", + "reqwest", + "rust-crypto", + "sea-orm", + "serde", + "serde_json", + "serde_yaml", + "solana-sdk", + "sqlx", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "vergen", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", + "yellowstone-grpc-tools", +] + [[package]] name = "das-ops" version = "0.7.2" @@ -3724,46 +3764,6 @@ dependencies = [ "url", ] -[[package]] -name = "nft_ingester2" -version = "0.7.2" -dependencies = [ - "anyhow", - "async-stream", - "atty", - "cargo-lock", - "chrono", - "clap 4.5.4", - "digital_asset_types", - "futures", - "git-version", - "hyper", - "json5", - "lazy_static", - "opentelemetry", - "opentelemetry-jaeger", - "opentelemetry_sdk", - "program_transformers", - "prometheus", - "redis 0.25.3", - "reqwest", - "rust-crypto", - "sea-orm", - "serde", - "serde_json", - "serde_yaml", - "solana-sdk", - "sqlx", - "tokio", - "tracing", - "tracing-opentelemetry", - "tracing-subscriber", - "vergen", - "yellowstone-grpc-client", - "yellowstone-grpc-proto", - "yellowstone-grpc-tools", -] - [[package]] name = "nix" version = "0.26.4" diff --git a/Cargo.toml b/Cargo.toml index a666f375f..e6e1ddc28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ "migration", "nft_ingester", "ops", - "nft_ingester2", + "grpc-ingest", "program_transformers", "tools/acc_forwarder", "tools/bgtask_creator", diff --git a/nft_ingester2/Cargo.toml b/grpc-ingest/Cargo.toml similarity index 98% rename from nft_ingester2/Cargo.toml rename to grpc-ingest/Cargo.toml index 51afb7b09..958bca6b4 100644 --- a/nft_ingester2/Cargo.toml +++ b/grpc-ingest/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "nft_ingester2" +name = "das-grpc-ingest" version = { workspace = true } edition = { workspace = true } repository = { workspace = true } diff --git a/nft_ingester2/README.md b/grpc-ingest/README.md similarity index 100% rename from nft_ingester2/README.md rename to grpc-ingest/README.md diff --git a/nft_ingester2/build.rs b/grpc-ingest/build.rs similarity index 100% rename from nft_ingester2/build.rs rename to grpc-ingest/build.rs diff --git a/nft_ingester2/config-download-metadata.yml b/grpc-ingest/config-download-metadata.yml similarity index 100% rename from nft_ingester2/config-download-metadata.yml rename to grpc-ingest/config-download-metadata.yml diff --git a/nft_ingester2/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml similarity index 100% rename from nft_ingester2/config-grpc2redis.yml rename to grpc-ingest/config-grpc2redis.yml diff --git a/nft_ingester2/config-ingester.yml b/grpc-ingest/config-ingester.yml similarity index 100% rename from nft_ingester2/config-ingester.yml rename to grpc-ingest/config-ingester.yml diff --git a/nft_ingester2/src/config.rs b/grpc-ingest/src/config.rs similarity index 100% rename from nft_ingester2/src/config.rs rename to grpc-ingest/src/config.rs diff --git a/nft_ingester2/src/download_metadata.rs b/grpc-ingest/src/download_metadata.rs similarity index 100% rename from nft_ingester2/src/download_metadata.rs rename to grpc-ingest/src/download_metadata.rs diff --git a/nft_ingester2/src/grpc.rs b/grpc-ingest/src/grpc.rs similarity index 100% rename from nft_ingester2/src/grpc.rs rename to grpc-ingest/src/grpc.rs diff --git a/nft_ingester2/src/ingester.rs b/grpc-ingest/src/ingester.rs similarity index 100% rename from nft_ingester2/src/ingester.rs rename to grpc-ingest/src/ingester.rs diff --git a/nft_ingester2/src/main.rs b/grpc-ingest/src/main.rs similarity index 100% rename from nft_ingester2/src/main.rs rename to grpc-ingest/src/main.rs diff --git a/nft_ingester2/src/postgres.rs b/grpc-ingest/src/postgres.rs similarity index 100% rename from nft_ingester2/src/postgres.rs rename to grpc-ingest/src/postgres.rs diff --git a/nft_ingester2/src/prom.rs b/grpc-ingest/src/prom.rs similarity index 100% rename from nft_ingester2/src/prom.rs rename to grpc-ingest/src/prom.rs diff --git a/nft_ingester2/src/redis.rs b/grpc-ingest/src/redis.rs similarity index 100% rename from nft_ingester2/src/redis.rs rename to grpc-ingest/src/redis.rs diff --git a/nft_ingester2/src/tracing.rs b/grpc-ingest/src/tracing.rs similarity index 100% rename from nft_ingester2/src/tracing.rs rename to grpc-ingest/src/tracing.rs diff --git a/nft_ingester2/src/util.rs b/grpc-ingest/src/util.rs similarity index 100% rename from nft_ingester2/src/util.rs rename to grpc-ingest/src/util.rs diff --git a/nft_ingester2/src/version.rs b/grpc-ingest/src/version.rs similarity index 100% rename from nft_ingester2/src/version.rs rename to grpc-ingest/src/version.rs From 294fa1a0c351e0bed0e826656842857ef68228c5 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 15 Apr 2024 13:24:27 +0200 Subject: [PATCH 21/72] fix: applying account and transction filters to grpc subscription request --- .github/workflows/test.yml | 2 +- grpc-ingest/README.md | 2 +- grpc-ingest/config-grpc2redis.yml | 18 ++++++++----- grpc-ingest/src/config.rs | 4 +-- grpc-ingest/src/grpc.rs | 25 +++++++++++++++--- nft_ingester/src/tasks/mod.rs | 4 +-- .../src/bubblegum/decompress.rs | 26 ------------------- program_transformers/src/lib.rs | 2 +- 8 files changed, 40 insertions(+), 43 deletions(-) delete mode 100644 program_transformers/src/bubblegum/decompress.rs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f4b4111d..60f1b69ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: run: | cargo tree git checkout Cargo.lock - cargo tree --frozen + cargo tree # fmt - name: Check fmt diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md index a2afa2226..d721d863b 100644 --- a/grpc-ingest/README.md +++ b/grpc-ingest/README.md @@ -30,4 +30,4 @@ psql: ``` PGPASSWORD=solana psql -h localhost -U solana -d solana -``` +``` \ No newline at end of file diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index 73ed908f4..a7e79d1dc 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -1,21 +1,25 @@ prometheus: 127.0.0.1:8873 endpoint: http://127.0.0.1:10000 x_token: null -commitment: processed +commitment: finalized accounts: stream: ACCOUNTS stream_maxlen: 100_000_000 stream_data_key: data - filters: - - owner: - - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA - - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s + filter: + owner: + - "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s" + - "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb" + - "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + - "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL" + - "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY" + - "CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d" transactions: stream: TRANSACTIONS stream_maxlen: 10_000_000 stream_data_key: data - filters: - - account_include: + filter: + account_include: - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY redis: url: redis://localhost:6379 diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index c3c45531e..90b7c005c 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -61,7 +61,7 @@ pub struct ConfigGrpcAccounts { #[serde(default = "ConfigGrpcAccounts::default_stream_data_key")] pub stream_data_key: String, - pub filters: Vec, + pub filter: ConfigGrpcRequestAccounts, } impl ConfigGrpcAccounts { @@ -90,7 +90,7 @@ pub struct ConfigGrpcTransactions { #[serde(default = "ConfigGrpcTransactions::default_stream_data_key")] pub stream_data_key: String, - pub filters: Vec, + pub filter: ConfigGrpcRequestTransactions, } impl ConfigGrpcTransactions { diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 01387c025..47cf9c718 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -5,6 +5,7 @@ use { anyhow::Context, futures::stream::StreamExt, redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, + std::collections::HashMap, std::{sync::Arc, time::Duration}, tokio::{ task::JoinSet, @@ -12,7 +13,10 @@ use { }, tracing::warn, yellowstone_grpc_client::GeyserGrpcClient, - yellowstone_grpc_proto::{prelude::subscribe_update::UpdateOneof, prost::Message}, + yellowstone_grpc_proto::{ + geyser::SubscribeRequest, prelude::subscribe_update::UpdateOneof, prost::Message, + }, + yellowstone_grpc_tools::config::GrpcRequestToProto, }; pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { @@ -41,7 +45,22 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { .connect() .await .context("failed to connect go gRPC")?; - let (mut _subscribe_tx, mut stream) = client.subscribe().await?; + + let mut accounts = HashMap::with_capacity(1); + let mut transactions = HashMap::with_capacity(1); + + accounts.insert("das".to_string(), config.accounts.filter.clone().to_proto()); + transactions.insert( + "das".to_string(), + config.transactions.filter.clone().to_proto(), + ); + + let request = SubscribeRequest { + accounts, + transactions, + ..Default::default() + }; + let (mut _subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; // recv-send loop let mut shutdown = create_shutdown()?; @@ -117,7 +136,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let result: RedisResult = pipe.atomic().query_async(&mut connection).await; - let status = if result.is_ok() { Ok(()) } else { Err(()) }; + let status = result.map(|_| ()).map_err(|_| ()); redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); diff --git a/nft_ingester/src/tasks/mod.rs b/nft_ingester/src/tasks/mod.rs index ec7a813fe..6c6f854b6 100644 --- a/nft_ingester/src/tasks/mod.rs +++ b/nft_ingester/src/tasks/mod.rs @@ -324,8 +324,8 @@ impl TaskManager { tokio::task::spawn(async move { while let Some(task) = receiver.recv().await { if let Some(task_created_time) = task.created_at { - let bus_time = - Utc::now().timestamp_millis() - task_created_time.timestamp_millis(); + let bus_time = Utc::now().timestamp_millis() + - task_created_time.and_utc().timestamp_millis(); metric! { statsd_histogram!("ingester.bgtask.bus_time", bus_time as u64, "type" => task.name); } diff --git a/program_transformers/src/bubblegum/decompress.rs b/program_transformers/src/bubblegum/decompress.rs deleted file mode 100644 index 208e9aabc..000000000 --- a/program_transformers/src/bubblegum/decompress.rs +++ /dev/null @@ -1,26 +0,0 @@ -use { - crate::{ - bubblegum::db::{ - upsert_asset_with_compression_info, upsert_asset_with_leaf_info_for_decompression, - }, - error::ProgramTransformerResult, - }, - blockbuster::{instruction::InstructionBundle, programs::bubblegum::BubblegumInstruction}, - sea_orm::{ConnectionTrait, TransactionTrait}, -}; - -pub async fn decompress<'c, T>( - _parsing_result: &BubblegumInstruction, - bundle: &InstructionBundle<'c>, - txn: &'c T, -) -> ProgramTransformerResult<()> -where - T: ConnectionTrait + TransactionTrait, -{ - let id_bytes = bundle.keys.get(3).unwrap().to_bytes().to_vec(); - - // Partial update of asset table with just leaf. - upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.clone()).await?; - upsert_asset_with_compression_info(txn, id_bytes.clone(), false, false, 1, Some(id_bytes), true) - .await -} diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index 6b6259d01..79c407bd4 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -53,7 +53,7 @@ pub struct TransactionInfo { pub meta_inner_instructions: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DownloadMetadataInfo { asset_data_id: Vec, uri: String, From 98a7e4e7e9f9979804ddd090d7dc6241e1330f73 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 2 May 2024 14:43:44 +0200 Subject: [PATCH 22/72] refactor: take out multiple connections to dragonmouth and then have the main thread keep a chache of seen events so that it doesnt push to ingest if seen by another connections --- Cargo.lock | 11 ++ Cargo.toml | 1 + grpc-ingest/Cargo.toml | 2 + grpc-ingest/src/config.rs | 32 ++++++ grpc-ingest/src/grpc.rs | 220 +++++++++++++++++++++++--------------- 5 files changed, 181 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40e3b3e72..f414c12c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,9 +1830,11 @@ dependencies = [ "digital_asset_types", "futures", "git-version", + "hex", "hyper", "json5", "lazy_static", + "lru", "opentelemetry", "opentelemetry-jaeger", "opentelemetry_sdk", @@ -3499,6 +3501,15 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lru" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +dependencies = [ + "hashbrown 0.14.2", +] + [[package]] name = "matchers" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e6e1ddc28..bdd86ea96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ jsonrpsee = "0.16.2" jsonrpsee-core = "0.16.2" lazy_static = "1.4.0" log = "0.4.17" +lru = "0.12.3" metrics = "0.20.1" migration = { path = "migration" } mime_guess = "2.0.4" diff --git a/grpc-ingest/Cargo.toml b/grpc-ingest/Cargo.toml index 958bca6b4..6ec9a99db 100644 --- a/grpc-ingest/Cargo.toml +++ b/grpc-ingest/Cargo.toml @@ -21,9 +21,11 @@ chrono = { workspace = true } clap = { workspace = true, features = ["cargo", "derive"] } digital_asset_types = { workspace = true } futures = { workspace = true } +hex = { workspace = true } hyper = { workspace = true, features = ["server"] } json5 = { workspace = true } lazy_static = { workspace = true } +lru = { workspace = true } opentelemetry = { workspace = true } opentelemetry-jaeger = { workspace = true, features = ["rt-tokio"] } opentelemetry_sdk = { workspace = true, features = ["trace"] } diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 90b7c005c..e7f205d17 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -46,9 +46,41 @@ pub struct ConfigGrpc { pub accounts: ConfigGrpcAccounts, pub transactions: ConfigGrpcTransactions, + #[serde( + default = "ConfigGrpc::default_geyser_client_connections", + deserialize_with = "deserialize_usize_str" + )] + pub geyser_client_connections: usize, + + #[serde( + default = "ConfigGrpc::default_geyser_update_message_buffer_size", + deserialize_with = "deserialize_usize_str" + )] + pub geyser_update_message_buffer_size: usize, + + #[serde( + default = "ConfigGrpc::solana_seen_event_cache_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub solana_seen_event_cache_max_size: usize, + pub redis: ConfigGrpcRedis, } +impl ConfigGrpc { + pub fn default_geyser_client_connections() -> usize { + 5 + } + + pub fn default_geyser_update_message_buffer_size() -> usize { + 100_000 + } + + pub fn solana_seen_event_cache_max_size() -> usize { + 10_000 + } +} + #[derive(Debug, Deserialize)] pub struct ConfigGrpcAccounts { #[serde(default = "ConfigGrpcAccounts::default_stream")] diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 47cf9c718..a17ad78f2 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -3,15 +3,18 @@ use { config::ConfigGrpc, prom::redis_xadd_status_inc, redis::metrics_xlen, util::create_shutdown, }, anyhow::Context, - futures::stream::StreamExt, + futures::{channel::mpsc, stream::StreamExt, SinkExt}, + hex, + lru::LruCache, redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, - std::collections::HashMap, - std::{sync::Arc, time::Duration}, + std::num::NonZeroUsize, + std::{collections::HashMap, sync::Arc, time::Duration}, tokio::{ + spawn, task::JoinSet, time::{sleep, Instant}, }, - tracing::warn, + tracing::{error, warn}, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::SubscribeRequest, prelude::subscribe_update::UpdateOneof, prost::Message, @@ -21,13 +24,14 @@ use { pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let config = Arc::new(config); + let (tx, mut rx) = mpsc::channel::(config.geyser_update_message_buffer_size); // Adjust buffer size as needed // Connect to Redis let client = redis::Client::open(config.redis.url.clone())?; let connection = client.get_multiplexed_tokio_connection().await?; // Check stream length for the metrics - let jh_metrics_xlen = tokio::spawn({ + let jh_metrics_xlen = spawn({ let connection = connection.clone(); let streams = vec![ config.accounts.stream.clone(), @@ -37,39 +41,59 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { }); tokio::pin!(jh_metrics_xlen); - // Create gRPC client, subscribe and handle messages - let mut client = GeyserGrpcClient::build_from_shared(config.endpoint.clone())? - .x_token(config.x_token.clone())? - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(10)) - .connect() - .await - .context("failed to connect go gRPC")?; - - let mut accounts = HashMap::with_capacity(1); - let mut transactions = HashMap::with_capacity(1); - - accounts.insert("das".to_string(), config.accounts.filter.clone().to_proto()); - transactions.insert( - "das".to_string(), - config.transactions.filter.clone().to_proto(), - ); + // Spawn gRPC client connections + for _ in 0..config.geyser_client_connections { + let config = Arc::clone(&config); + let mut tx = tx.clone(); - let request = SubscribeRequest { - accounts, - transactions, - ..Default::default() - }; - let (mut _subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; + let mut client = GeyserGrpcClient::build_from_shared(config.endpoint.clone())? + .x_token(config.x_token.clone())? + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(10)) + .connect() + .await + .context("failed to connect to gRPC")?; + + spawn(async move { + let mut accounts = HashMap::with_capacity(1); + let mut transactions = HashMap::with_capacity(1); + + accounts.insert("das".to_string(), config.accounts.filter.clone().to_proto()); + transactions.insert( + "das".to_string(), + config.transactions.filter.clone().to_proto(), + ); + + let request = SubscribeRequest { + accounts, + transactions, + ..Default::default() + }; + let (_subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; + + while let Some(Ok(msg)) = stream.next().await { + if let Some(update) = msg.update_oneof { + tx.send(update) + .await + .expect("Failed to send update to management thread"); + } + } + Ok::<(), anyhow::Error>(()) + }); + } - // recv-send loop + // Management thread let mut shutdown = create_shutdown()?; + let mut tasks = JoinSet::new(); let mut pipe = redis::pipe(); let mut pipe_accounts = 0; let mut pipe_transactions = 0; let deadline = sleep(config.redis.pipeline_max_idle); tokio::pin!(deadline); - let mut tasks = JoinSet::new(); + + let mut seen_update_events = LruCache::::new( + NonZeroUsize::new(config.solana_seen_event_cache_max_size).expect("Non zero value"), + ); let result = loop { tokio::select! { @@ -82,67 +106,93 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { warn!("{signal} received, waiting spawned tasks..."); break Ok(()); }, - msg = stream.next() => { - match msg { - Some(Ok(msg)) => match msg.update_oneof { - Some(UpdateOneof::Account(account)) => { - pipe.xadd_maxlen( - &config.accounts.stream, - StreamMaxlen::Approx(config.accounts.stream_maxlen), - "*", - &[(&config.accounts.stream_data_key, account.encode_to_vec())], - ); - pipe_accounts += 1; + Some(update) = rx.next() => { + match update { + UpdateOneof::Account(account) => { + let slot_pubkey = format!("{}:{}", account.slot, hex::encode(account.account.as_ref().map(|account| account.pubkey.clone()).unwrap_or_default())); + + if seen_update_events.get(&slot_pubkey).is_some() { + continue; + } else { + seen_update_events.put(slot_pubkey, ()); + }; + + pipe.xadd_maxlen( + &config.accounts.stream, + StreamMaxlen::Approx(config.accounts.stream_maxlen), + "*", + &[(&config.accounts.stream_data_key, account.encode_to_vec())], + ); + + pipe_accounts += 1; + } + UpdateOneof::Transaction(transaction) => { + let slot_signature = format!("{}:{}", transaction.slot, hex::encode(transaction.transaction.as_ref().map(|t| t.signature.clone()).unwrap_or_default())); + + if seen_update_events.get(&slot_signature).is_some() { + continue; + } else { + seen_update_events.put(slot_signature, ()); + }; + + pipe.xadd_maxlen( + &config.transactions.stream, + StreamMaxlen::Approx(config.transactions.stream_maxlen), + "*", + &[(&config.transactions.stream_data_key, transaction.encode_to_vec())] + ); + + pipe_transactions += 1; + } + _ => continue, + } + if pipe_accounts + pipe_transactions >= config.redis.pipeline_max_size { + let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); + let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); + let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); + deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); + + tasks.spawn({ + let mut connection = connection.clone(); + let config = Arc::clone(&config); + async move { + let result: RedisResult = + pipe.atomic().query_async(&mut connection).await; + + let status = result.map(|_| ()).map_err(|_| ()); + redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); + redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); + + Ok::<(), anyhow::Error>(()) } - Some(UpdateOneof::Slot(_)) => continue, - Some(UpdateOneof::Transaction(transaction)) => { - pipe.xadd_maxlen( - &config.transactions.stream, - StreamMaxlen::Approx(config.transactions.stream_maxlen), - "*", - &[(&config.transactions.stream_data_key, transaction.encode_to_vec())] - ); - pipe_transactions += 1; + }); + } + }, + _ = &mut deadline => { + if pipe_accounts + pipe_transactions > 0 { + let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); + let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); + let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); + deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); + + tasks.spawn({ + let mut connection = connection.clone(); + let config = Arc::clone(&config); + async move { + let result: RedisResult = + pipe.atomic().query_async(&mut connection).await; + + let status = result.map(|_| ()).map_err(|_| ()); + redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); + redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); + + Ok::<(), anyhow::Error>(()) } - Some(UpdateOneof::Block(_)) => continue, - Some(UpdateOneof::Ping(_)) => continue, - Some(UpdateOneof::Pong(_)) => continue, - Some(UpdateOneof::BlockMeta(_)) => continue, - Some(UpdateOneof::Entry(_)) => continue, - Some(UpdateOneof::TransactionStatus(_)) => continue, - None => break Err(anyhow::anyhow!("received invalid update gRPC message")), - }, - Some(Err(error)) => break Err(error.into()), - None => break Err(anyhow::anyhow!("geyser gRPC request is finished")), - }; - if pipe_accounts + pipe_transactions < config.redis.pipeline_max_size { - continue; + }); } }, - _ = &mut deadline => {}, }; - let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); - let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); - let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); - deadline - .as_mut() - .reset(Instant::now() + config.redis.pipeline_max_idle); - - tasks.spawn({ - let mut connection = connection.clone(); - let config = Arc::clone(&config); - async move { - let result: RedisResult = - pipe.atomic().query_async(&mut connection).await; - - let status = result.map(|_| ()).map_err(|_| ()); - redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); - redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); - - Ok::<(), anyhow::Error>(()) - } - }); while tasks.len() >= config.redis.max_xadd_in_process { if let Some(result) = tasks.join_next().await { result??; From 997d515d6551fe0cdaed6a04513ccc57b72f3bbb Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 3 May 2024 12:15:21 +0200 Subject: [PATCH 23/72] refactor: switch to accepting multiple dragonmouth endoints to mimic dual validators writing to ingest stream --- grpc-ingest/config-grpc2redis.yml | 3 ++- grpc-ingest/src/config.rs | 11 +---------- grpc-ingest/src/grpc.rs | 7 ++++--- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index a7e79d1dc..36a6b5fbe 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -1,5 +1,6 @@ prometheus: 127.0.0.1:8873 -endpoint: http://127.0.0.1:10000 +geyser_endpoints: + - http://127.0.0.1:10000 x_token: null commitment: finalized accounts: diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index e7f205d17..174f6ad8d 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -39,18 +39,13 @@ pub struct ConfigPrometheus { #[derive(Debug, Deserialize)] pub struct ConfigGrpc { - pub endpoint: String, pub x_token: Option, pub commitment: ConfigGrpcRequestCommitment, pub accounts: ConfigGrpcAccounts, pub transactions: ConfigGrpcTransactions, - #[serde( - default = "ConfigGrpc::default_geyser_client_connections", - deserialize_with = "deserialize_usize_str" - )] - pub geyser_client_connections: usize, + pub geyser_endpoints: Vec, #[serde( default = "ConfigGrpc::default_geyser_update_message_buffer_size", @@ -68,10 +63,6 @@ pub struct ConfigGrpc { } impl ConfigGrpc { - pub fn default_geyser_client_connections() -> usize { - 5 - } - pub fn default_geyser_update_message_buffer_size() -> usize { 100_000 } diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index a17ad78f2..4fd4102de 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -42,11 +42,11 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { tokio::pin!(jh_metrics_xlen); // Spawn gRPC client connections - for _ in 0..config.geyser_client_connections { + for endpoint in config.geyser_endpoints.clone() { let config = Arc::clone(&config); let mut tx = tx.clone(); - let mut client = GeyserGrpcClient::build_from_shared(config.endpoint.clone())? + let mut client = GeyserGrpcClient::build_from_shared(endpoint)? .x_token(config.x_token.clone())? .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10)) @@ -69,6 +69,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { transactions, ..Default::default() }; + let (_subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; while let Some(Ok(msg)) = stream.next().await { @@ -127,7 +128,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { pipe_accounts += 1; } UpdateOneof::Transaction(transaction) => { - let slot_signature = format!("{}:{}", transaction.slot, hex::encode(transaction.transaction.as_ref().map(|t| t.signature.clone()).unwrap_or_default())); + let slot_signature = hex::encode(transaction.transaction.as_ref().map(|t| t.signature.clone()).unwrap_or_default()).to_string(); if seen_update_events.get(&slot_signature).is_some() { continue; From 67cc21e304eeaa626e7dc20ef0a3b45a4516b853 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 9 May 2024 11:50:33 +0200 Subject: [PATCH 24/72] fix: clippy errors --- grpc-ingest/src/config.rs | 4 ++-- grpc-ingest/src/grpc.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 174f6ad8d..151dfbe08 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -63,11 +63,11 @@ pub struct ConfigGrpc { } impl ConfigGrpc { - pub fn default_geyser_update_message_buffer_size() -> usize { + pub const fn default_geyser_update_message_buffer_size() -> usize { 100_000 } - pub fn solana_seen_event_cache_max_size() -> usize { + pub const fn solana_seen_event_cache_max_size() -> usize { 10_000 } } diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 4fd4102de..11f9dddaa 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -4,7 +4,6 @@ use { }, anyhow::Context, futures::{channel::mpsc, stream::StreamExt, SinkExt}, - hex, lru::LruCache, redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, std::num::NonZeroUsize, @@ -14,7 +13,7 @@ use { task::JoinSet, time::{sleep, Instant}, }, - tracing::{error, warn}, + tracing::warn, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::SubscribeRequest, prelude::subscribe_update::UpdateOneof, prost::Message, From 5f08298ac44cba856713eaf2313ed4f595e07c25 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 15 May 2024 11:35:39 +0200 Subject: [PATCH 25/72] fix: processes transactions for a tree sequentially to make sure there is on out of order processing. process the transaction by ops instead of queueing to TXN_FILL redis stream. --- Cargo.lock | 4 + core/src/db.rs | 2 +- core/src/metrics.rs | 4 +- core/src/plerkle_messenger_queue.rs | 2 +- core/src/solana_rpc.rs | 4 +- ops/Cargo.toml | 4 + ops/src/account/program.rs | 4 +- ops/src/account/single.rs | 4 +- ops/src/bubblegum/audit.rs | 5 +- ops/src/bubblegum/backfiller.rs | 409 +++++++++++++++++++--------- ops/src/bubblegum/cmd.rs | 21 ++ ops/src/bubblegum/tree.rs | 56 ++-- 12 files changed, 337 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c294f301a..a2e20cfdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1738,6 +1738,7 @@ dependencies = [ "anyhow", "backon", "borsh 0.10.3", + "bs58 0.4.0", "cadence", "cadence-macros", "clap 4.4.8", @@ -1752,12 +1753,15 @@ dependencies = [ "mpl-bubblegum", "plerkle_messenger", "plerkle_serialization", + "program_transformers", "sea-orm", "solana-account-decoder", "solana-client", + "solana-program", "solana-sdk", "solana-transaction-status", "spl-account-compression", + "sqlx", "thiserror", "tokio", ] diff --git a/core/src/db.rs b/core/src/db.rs index 3037f670d..5c211a877 100644 --- a/core/src/db.rs +++ b/core/src/db.rs @@ -27,7 +27,7 @@ pub struct PoolArgs { ///// # Returns ///// ///// * `Result` - On success, returns a `DatabaseConnection`. On failure, returns a `DbErr`. -pub async fn connect_db(config: PoolArgs) -> Result { +pub async fn connect_db(config: &PoolArgs) -> Result { let options: PgConnectOptions = config.database_url.parse()?; PgPoolOptions::new() diff --git a/core/src/metrics.rs b/core/src/metrics.rs index 9c0d3c531..13bf4e261 100644 --- a/core/src/metrics.rs +++ b/core/src/metrics.rs @@ -14,8 +14,8 @@ pub struct MetricsArgs { pub metrics_prefix: String, } -pub fn setup_metrics(config: MetricsArgs) -> Result<()> { - let host = (config.metrics_host, config.metrics_port); +pub fn setup_metrics(config: &MetricsArgs) -> Result<()> { + let host = (config.metrics_host.clone(), config.metrics_port); let socket = UdpSocket::bind("0.0.0.0:0")?; socket.set_nonblocking(true)?; diff --git a/core/src/plerkle_messenger_queue.rs b/core/src/plerkle_messenger_queue.rs index 8c30d01c0..11b1166ed 100644 --- a/core/src/plerkle_messenger_queue.rs +++ b/core/src/plerkle_messenger_queue.rs @@ -65,7 +65,7 @@ pub struct QueuePool { } impl QueuePool { - pub async fn try_from_config(config: QueueArgs) -> anyhow::Result { + pub async fn try_from_config(config: &QueueArgs) -> anyhow::Result { let size = usize::try_from(config.messenger_queue_connections)?; let (tx, rx) = mpsc::channel(size); diff --git a/core/src/solana_rpc.rs b/core/src/solana_rpc.rs index 71f86f8a6..b517f1404 100644 --- a/core/src/solana_rpc.rs +++ b/core/src/solana_rpc.rs @@ -31,8 +31,8 @@ pub struct SolanaRpcArgs { pub struct Rpc(Arc); impl Rpc { - pub fn from_config(config: SolanaRpcArgs) -> Self { - Rpc(Arc::new(RpcClient::new(config.solana_rpc_url))) + pub fn from_config(config: &SolanaRpcArgs) -> Self { + Rpc(Arc::new(RpcClient::new(config.solana_rpc_url.clone()))) } pub async fn get_transaction( diff --git a/ops/Cargo.toml b/ops/Cargo.toml index e6cb7a870..13cccc30b 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -13,6 +13,7 @@ anchor-client = { workspace = true } anyhow = { workspace = true } backon = { workspace = true } borsh = { workspace = true } +bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "env"] } @@ -27,11 +28,14 @@ log = { workspace = true } mpl-bubblegum = { workspace = true } plerkle_messenger = { workspace = true } plerkle_serialization = { workspace = true } +program_transformers = { workspace = true } sea-orm = { workspace = true } solana-account-decoder = { workspace = true } solana-client = { workspace = true } +solana-program = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true } +sqlx = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/ops/src/account/program.rs b/ops/src/account/program.rs index 7dd990582..f3ff38736 100644 --- a/ops/src/account/program.rs +++ b/ops/src/account/program.rs @@ -37,8 +37,8 @@ fn parse_pubkey(s: &str) -> Result { } pub async fn run(config: Args) -> Result<()> { - let rpc = Rpc::from_config(config.solana); - let queue = QueuePool::try_from_config(config.queue).await?; + let rpc = Rpc::from_config(&config.solana); + let queue = QueuePool::try_from_config(&config.queue).await?; let accounts = rpc.get_program_accounts(&config.program, None).await?; diff --git a/ops/src/account/single.rs b/ops/src/account/single.rs index 41269bb05..b40161fd2 100644 --- a/ops/src/account/single.rs +++ b/ops/src/account/single.rs @@ -32,8 +32,8 @@ fn parse_pubkey(s: &str) -> Result { } pub async fn run(config: Args) -> Result<()> { - let rpc = Rpc::from_config(config.solana); - let queue = QueuePool::try_from_config(config.queue).await?; + let rpc = Rpc::from_config(&config.solana); + let queue = QueuePool::try_from_config(&config.queue).await?; let AccountDetails { account, diff --git a/ops/src/bubblegum/audit.rs b/ops/src/bubblegum/audit.rs index 4da0a2652..deee3c4dd 100644 --- a/ops/src/bubblegum/audit.rs +++ b/ops/src/bubblegum/audit.rs @@ -27,10 +27,11 @@ pub struct Args { pub batch_size: u64, } +/// The `audit` commands checks `cl_audits_v2` for any failed transactions and logs them to stdout. pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(config.database).await?; + let pool = connect_db(&config.database).await?; - let solana_rpc = Rpc::from_config(config.solana); + let solana_rpc = Rpc::from_config(&config.solana); let mut output = stdout(); let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); diff --git a/ops/src/bubblegum/backfiller.rs b/ops/src/bubblegum/backfiller.rs index 8b0e5ed4f..fbfb3bf7c 100644 --- a/ops/src/bubblegum/backfiller.rs +++ b/ops/src/bubblegum/backfiller.rs @@ -1,20 +1,27 @@ -use super::tree::{TreeErrorKind, TreeGapFill, TreeGapModel, TreeResponse}; +use super::{ + tree::{TreeGapFill, TreeGapModel, TreeResponse}, + BubblegumOpsErrorKind, +}; use anyhow::Result; use cadence_macros::{statsd_count, statsd_time}; use clap::Parser; -use das_core::{ - connect_db, setup_metrics, MetricsArgs, PoolArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs, -}; +use das_core::{connect_db, setup_metrics, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; use digital_asset_types::dao::cl_audits_v2; -use flatbuffers::FlatBufferBuilder; +use futures::future::{ready, FutureExt}; use futures::{stream::FuturesUnordered, StreamExt}; use indicatif::HumanDuration; -use log::{error, info}; -use plerkle_serialization::serializer::seralize_encoded_transaction_with_status; -use sea_orm::{ - ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector, -}; +use log::{debug, error}; +use program_transformers::{ProgramTransformer, TransactionInfo}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector}; +use solana_program::pubkey::Pubkey; +use solana_sdk::instruction::CompiledInstruction; use solana_sdk::signature::Signature; +use solana_sdk::transaction::VersionedTransaction; +use solana_transaction_status::{ + option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta, + InnerInstruction, InnerInstructions, UiInstruction, +}; +use sqlx::PgPool; use std::time::Instant; use tokio::{sync::mpsc, task::JoinHandle}; @@ -25,7 +32,7 @@ pub struct Args { pub tree_crawler_count: usize, /// The size of the signature channel. - #[arg(long, env, default_value = "10000")] + #[arg(long, env, default_value = "100000")] pub signature_channel_size: usize, /// The size of the signature channel. @@ -33,8 +40,8 @@ pub struct Args { pub gap_channel_size: usize, /// The number of transaction workers. - #[arg(long, env, default_value = "100")] - pub transaction_worker_count: usize, + #[arg(long, env, default_value = "50")] + pub transaction_worker_count_per_tree: usize, /// The number of gap workers. #[arg(long, env, default_value = "25")] @@ -48,10 +55,6 @@ pub struct Args { #[clap(flatten)] pub database: PoolArgs, - /// Redis configuration - #[clap(flatten)] - pub queue: QueueArgs, - /// Metrics configuration #[clap(flatten)] pub metrics: MetricsArgs, @@ -61,103 +64,53 @@ pub struct Args { pub solana: SolanaRpcArgs, } -/// Runs the backfilling process for the tree crawler. +/// Executes the backfilling operation for the tree crawler. /// -/// This function initializes the necessary components for the backfilling process, -/// including database connections, RPC clients, and worker managers for handling -/// transactions and gaps. It then proceeds to fetch the trees that need to be crawled -/// and manages the crawling process across multiple workers. +/// This function sets up the essential components required for the backfilling operation, +/// including database connections, RPC clients, and worker managers to handle +/// transactions and gaps. It retrieves the necessary trees for crawling and orchestrates +/// the crawling operation across various workers. /// -/// The function handles the following major tasks: -/// - Establishing connections to the database and initializing RPC clients. -/// - Setting up channels for communication between different parts of the system. -/// - Spawning worker managers for processing transactions and gaps. -/// - Fetching trees from the database and managing their crawling process. -/// - Reporting metrics and logging information throughout the process. +/// The function undertakes the following key tasks: +/// - Establishes database connections and initializes RPC clients. +/// - Configures channels for inter-component communication. +/// - Deploys worker managers to handle transactions and gaps. +/// - Retrieves trees from the database and oversees their crawling. +/// - Monitors metrics and logs activities throughout the operation. /// /// # Arguments /// -/// * `config` - A configuration object containing settings for the backfilling process, -/// including database, RPC, and worker configurations. +/// * `config` - A configuration object that includes settings for the backfilling operation, +/// such as database, RPC, and worker configurations. /// /// # Returns /// -/// This function returns a `Result` which is `Ok` if the backfilling process completes -/// successfully, or an `Err` with an appropriate error message if any part of the process -/// fails. +/// This function returns a `Result` which is `Ok` if the backfilling operation is completed +/// successfully, or an `Err` with a relevant error message if any part of the operation +/// encounters issues. /// /// # Errors /// -/// This function can return errors related to database connectivity, RPC failures, -/// or issues with spawning and managing worker tasks. +/// Potential errors can arise from database connectivity issues, RPC failures, +/// or complications in spawning and managing worker tasks. pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(config.database).await?; - - let solana_rpc = Rpc::from_config(config.solana); - let transaction_solana_rpc = solana_rpc.clone(); - let gap_solana_rpc = solana_rpc.clone(); - - setup_metrics(config.metrics)?; - - let (sig_sender, mut sig_receiver) = mpsc::channel::(config.signature_channel_size); - let gap_sig_sender = sig_sender.clone(); - let (gap_sender, mut gap_receiver) = mpsc::channel::(config.gap_channel_size); - - let queue = QueuePool::try_from_config(config.queue).await?; - - let transaction_worker_count = config.transaction_worker_count; - - let transaction_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - - while let Some(signature) = sig_receiver.recv().await { - if handlers.len() >= transaction_worker_count { - handlers.next().await; - } - - let solana_rpc = transaction_solana_rpc.clone(); - let queue = queue.clone(); - - let handle = spawn_transaction_worker(solana_rpc, queue, signature); - - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - }); - - let gap_worker_count = config.gap_worker_count; - - let gap_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - - while let Some(gap) = gap_receiver.recv().await { - if handlers.len() >= gap_worker_count { - handlers.next().await; - } - - let client = gap_solana_rpc.clone(); - let sender = gap_sig_sender.clone(); + let pool = connect_db(&config.database).await?; - let handle = spawn_crawl_worker(client, sender, gap); + let solana_rpc = Rpc::from_config(&config.solana); - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - }); + setup_metrics(&config.metrics)?; let started = Instant::now(); - let trees = if let Some(only_trees) = config.only_trees { - TreeResponse::find(&solana_rpc, only_trees).await? + let trees = if let Some(ref only_trees) = config.only_trees { + TreeResponse::find(&solana_rpc, only_trees.clone()).await? } else { TreeResponse::all(&solana_rpc).await? }; let tree_count = trees.len(); - info!( + debug!( "fetched {} trees in {}", tree_count, HumanDuration(started.elapsed()) @@ -171,29 +124,19 @@ pub async fn run(config: Args) -> Result<()> { crawl_handles.next().await; } - let sender = gap_sender.clone(); let pool = pool.clone(); - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + let solana_rpc = solana_rpc.clone(); - let handle = spawn_gap_worker(conn, sender, tree); + let handle = spawn_tree_worker(&config, pool, solana_rpc, tree); crawl_handles.push(handle); } futures::future::try_join_all(crawl_handles).await?; - drop(gap_sender); - info!("crawled all trees"); - - gap_worker_manager.await?; - drop(sig_sender); - info!("all gaps processed"); - - transaction_worker_manager.await?; - info!("all transactions queued"); statsd_time!("job.completed", started.elapsed()); - info!( + debug!( "crawled {} trees in {}", tree_count, HumanDuration(started.elapsed()) @@ -202,14 +145,96 @@ pub async fn run(config: Args) -> Result<()> { Ok(()) } -fn spawn_gap_worker( - conn: DatabaseConnection, - sender: mpsc::Sender, +fn spawn_tree_worker( + config: &Args, + pool: PgPool, + rpc: Rpc, tree: TreeResponse, ) -> JoinHandle> { + let config = config.clone(); + let gap_solana_rpc = rpc.clone(); + let gap_pool = pool.clone(); + tokio::spawn(async move { let timing = Instant::now(); + let transaction_worker_count = config.transaction_worker_count_per_tree; + + let (sig_sender, mut sig_receiver) = + mpsc::channel::(config.signature_channel_size); + let gap_sig_sender = sig_sender.clone(); + + let (gap_sender, mut gap_receiver) = mpsc::channel::(config.gap_channel_size); + let (transaction_sender, mut transaction_receiver) = + mpsc::channel::(config.signature_channel_size); + + let signature_worker_manager = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + + while let Some(signature) = sig_receiver.recv().await { + if handlers.len() >= transaction_worker_count { + handlers.next().await; + } + + let solana_rpc = rpc.clone(); + let transaction_sender = transaction_sender.clone(); + + let handle = spawn_transaction_worker(solana_rpc, transaction_sender, signature); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + + drop(transaction_sender); + }); + + let gap_worker_count = config.gap_worker_count; + + let gap_worker_manager = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + let sender = gap_sig_sender.clone(); + + while let Some(gap) = gap_receiver.recv().await { + if handlers.len() >= gap_worker_count { + handlers.next().await; + } + + let client = gap_solana_rpc.clone(); + let sender = sender.clone(); + + let handle = spawn_crawl_worker(client, sender, gap); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + + drop(sig_sender); + }); + + let transaction_worker_manager = tokio::spawn(async move { + let mut transactions = Vec::new(); + let pool = pool.clone(); + + let program_transformer = + ProgramTransformer::new(pool, Box::new(|_info| ready(Ok(())).boxed()), true); + + while let Some(gap) = transaction_receiver.recv().await { + transactions.push(gap); + } + + transactions.sort_by(|a, b| b.signature.cmp(&a.signature)); + + for transaction in transactions { + if let Err(e) = program_transformer.handle_transaction(&transaction).await { + error!("handle transaction: {:?}", e) + }; + } + }); + + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(gap_pool); + let mut gaps = TreeGapModel::find(&conn, tree.pubkey) .await? .into_iter() @@ -228,42 +253,35 @@ fn spawn_gap_worker( .one(&conn) .await?; + drop(conn); + if let Some(upper_seq) = upper_known_seq { let signature = Signature::try_from(upper_seq.tx.as_ref())?; - info!( - "tree {} has known highest seq {} filling tree from {}", - tree.pubkey, upper_seq.seq, signature - ); + gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); } else if tree.seq > 0 { - info!( - "tree {} has no known highest seq but the actual seq is {} filling whole tree", - tree.pubkey, tree.seq - ); gaps.push(TreeGapFill::new(tree.pubkey, None, None)); } if let Some(lower_seq) = lower_known_seq.filter(|seq| seq.seq > 1) { let signature = Signature::try_from(lower_seq.tx.as_ref())?; - info!( - "tree {} has known lowest seq {} filling tree starting at {}", - tree.pubkey, lower_seq.seq, signature - ); - gaps.push(TreeGapFill::new(tree.pubkey, Some(signature), None)); } - let gap_count = gaps.len(); - for gap in gaps { - if let Err(e) = sender.send(gap).await { + if let Err(e) = gap_sender.send(gap).await { statsd_count!("gap.failed", 1); error!("send gap: {:?}", e); } } - info!("crawling tree {} with {} gaps", tree.pubkey, gap_count); + drop(gap_sender); + gap_worker_manager.await?; + + signature_worker_manager.await?; + + transaction_worker_manager.await?; statsd_count!("tree.succeeded", 1); statsd_time!("tree.crawled", timing.elapsed()); @@ -292,27 +310,137 @@ fn spawn_crawl_worker( }) } +pub struct FetchedEncodedTransactionWithStatusMeta(pub EncodedConfirmedTransactionWithStatusMeta); + +impl TryFrom for TransactionInfo { + type Error = BubblegumOpsErrorKind; + + fn try_from( + fetched_transaction: FetchedEncodedTransactionWithStatusMeta, + ) -> Result { + let mut account_keys = Vec::new(); + let encoded_transaction_with_status_meta = fetched_transaction.0; + + let ui_transaction: VersionedTransaction = encoded_transaction_with_status_meta + .transaction + .transaction + .decode() + .ok_or(BubblegumOpsErrorKind::Generic( + "unable to decode transaction".to_string(), + ))?; + + let signature = ui_transaction.signatures[0]; + + let msg = ui_transaction.message; + + let meta = encoded_transaction_with_status_meta + .transaction + .meta + .ok_or(BubblegumOpsErrorKind::Generic( + "unable to get meta from transaction".to_string(), + ))?; + + for address in msg.static_account_keys().iter().copied() { + account_keys.push(address); + } + let ui_loaded_addresses = meta.loaded_addresses; + + let message_address_table_lookup = msg.address_table_lookups(); + + if message_address_table_lookup.is_some() { + if let OptionSerializer::Some(ui_lookup_table) = ui_loaded_addresses { + for address in ui_lookup_table.writable { + account_keys.push(PubkeyString(address).try_into()?); + } + + for address in ui_lookup_table.readonly { + account_keys.push(PubkeyString(address).try_into()?); + } + } + } + + let mut meta_inner_instructions = Vec::new(); + + let compiled_instruction = msg.instructions().to_vec(); + + let mut instructions = Vec::new(); + + for inner in compiled_instruction { + instructions.push(InnerInstruction { + stack_height: Some(0), + instruction: CompiledInstruction { + program_id_index: inner.program_id_index, + accounts: inner.accounts, + data: inner.data, + }, + }); + } + + meta_inner_instructions.push(InnerInstructions { + index: 0, + instructions, + }); + + if let OptionSerializer::Some(inner_instructions) = meta.inner_instructions { + for ix in inner_instructions { + let mut instructions = Vec::new(); + + for inner in ix.instructions { + if let UiInstruction::Compiled(compiled) = inner { + instructions.push(InnerInstruction { + stack_height: compiled.stack_height, + instruction: CompiledInstruction { + program_id_index: compiled.program_id_index, + accounts: compiled.accounts, + data: bs58::decode(compiled.data) + .into_vec() + .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?, + }, + }); + } + } + + meta_inner_instructions.push(InnerInstructions { + index: ix.index, + instructions, + }); + } + } + + Ok(Self { + slot: encoded_transaction_with_status_meta.slot, + account_keys, + signature, + message_instructions: msg.instructions().to_vec(), + meta_inner_instructions, + }) + } +} + async fn queue_transaction<'a>( client: Rpc, - queue: QueuePool, + sender: mpsc::Sender, signature: Signature, -) -> Result<(), TreeErrorKind> { +) -> Result<(), BubblegumOpsErrorKind> { let transaction = client.get_transaction(&signature).await?; - let message = seralize_encoded_transaction_with_status(FlatBufferBuilder::new(), transaction)?; - - queue - .push_transaction_backfill(message.finished_data()) - .await?; + sender + .send(FetchedEncodedTransactionWithStatusMeta(transaction).try_into()?) + .await + .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?; Ok(()) } -fn spawn_transaction_worker(client: Rpc, queue: QueuePool, signature: Signature) -> JoinHandle<()> { +fn spawn_transaction_worker( + client: Rpc, + sender: mpsc::Sender, + signature: Signature, +) -> JoinHandle<()> { tokio::spawn(async move { let timing = Instant::now(); - if let Err(e) = queue_transaction(client, queue, signature).await { + if let Err(e) = queue_transaction(client, sender, signature).await { error!("queue transaction: {:?}", e); statsd_count!("transaction.failed", 1); @@ -323,3 +451,18 @@ fn spawn_transaction_worker(client: Rpc, queue: QueuePool, signature: Signature) statsd_time!("transaction.queued", timing.elapsed()); }) } + +pub struct PubkeyString(pub String); + +impl TryFrom for Pubkey { + type Error = BubblegumOpsErrorKind; + + fn try_from(value: PubkeyString) -> Result { + let decoded_bytes = bs58::decode(value.0) + .into_vec() + .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?; + + Pubkey::try_from(decoded_bytes) + .map_err(|_| BubblegumOpsErrorKind::Generic("unable to convert pubkey".to_string())) + } +} diff --git a/ops/src/bubblegum/cmd.rs b/ops/src/bubblegum/cmd.rs index bb2244167..b19715483 100644 --- a/ops/src/bubblegum/cmd.rs +++ b/ops/src/bubblegum/cmd.rs @@ -1,6 +1,27 @@ use super::{audit, backfiller}; use anyhow::Result; use clap::{Args, Subcommand}; +use thiserror::Error as ThisError; + +#[derive(ThisError, Debug)] +pub enum BubblegumOpsErrorKind { + #[error("anchor")] + Anchor(#[from] anchor_client::anchor_lang::error::Error), + #[error("solana rpc")] + Rpc(#[from] solana_client::client_error::ClientError), + #[error("parse pubkey")] + ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), + #[error("serialize tree response")] + SerializeTreeResponse, + #[error("sea orm")] + Database(#[from] sea_orm::DbErr), + #[error("try from pubkey")] + TryFromPubkey, + #[error("try from signature")] + TryFromSignature, + #[error("generic error: {0}")] + Generic(String), +} #[derive(Debug, Clone, Subcommand)] pub enum Commands { diff --git a/ops/src/bubblegum/tree.rs b/ops/src/bubblegum/tree.rs index 09a3c92cb..14685ba40 100644 --- a/ops/src/bubblegum/tree.rs +++ b/ops/src/bubblegum/tree.rs @@ -1,8 +1,8 @@ +use super::BubblegumOpsErrorKind; use anyhow::Result; use borsh::BorshDeserialize; use clap::Args; -use das_core::{QueuePoolError, Rpc}; -use log::error; +use das_core::Rpc; use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; use solana_client::rpc_filter::{Memcmp, RpcFilterType}; use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; @@ -12,7 +12,6 @@ use spl_account_compression::state::{ merkle_tree_get_size, ConcurrentMerkleTreeHeader, CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1, }; use std::str::FromStr; -use thiserror::Error as ThisError; use tokio::sync::mpsc::Sender; const GET_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1000; @@ -24,30 +23,6 @@ pub struct ConfigBackfiller { pub solana_rpc_url: String, } -#[derive(ThisError, Debug)] -pub enum TreeErrorKind { - #[error("solana rpc")] - Rpc(#[from] solana_client::client_error::ClientError), - #[error("anchor")] - Achor(#[from] anchor_client::anchor_lang::error::Error), - #[error("perkle serialize")] - PerkleSerialize(#[from] plerkle_serialization::error::PlerkleSerializationError), - #[error("perkle messenger")] - PlerkleMessenger(#[from] plerkle_messenger::MessengerError), - #[error("queue pool")] - QueuePool(#[from] QueuePoolError), - #[error("parse pubkey")] - ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), - #[error("serialize tree response")] - SerializeTreeResponse, - #[error("sea orm")] - Database(#[from] sea_orm::DbErr), - #[error("try from pubkey")] - TryFromPubkey, - #[error("try from signature")] - TryFromSignature, -} - const TREE_GAP_SQL: &str = r#" WITH sequenced_data AS ( SELECT @@ -96,7 +71,10 @@ pub struct TreeGapModel { } impl TreeGapModel { - pub async fn find(conn: &DatabaseConnection, tree: Pubkey) -> Result, TreeErrorKind> { + pub async fn find( + conn: &DatabaseConnection, + tree: Pubkey, + ) -> Result, BubblegumOpsErrorKind> { let statement = Statement::from_sql_and_values( DbBackend::Postgres, TREE_GAP_SQL, @@ -111,14 +89,15 @@ impl TreeGapModel { } impl TryFrom for TreeGapFill { - type Error = TreeErrorKind; + type Error = BubblegumOpsErrorKind; fn try_from(model: TreeGapModel) -> Result { - let tree = Pubkey::try_from(model.tree).map_err(|_| TreeErrorKind::TryFromPubkey)?; + let tree = + Pubkey::try_from(model.tree).map_err(|_| BubblegumOpsErrorKind::TryFromPubkey)?; let upper = Signature::try_from(model.upper_bound_tx) - .map_err(|_| TreeErrorKind::TryFromSignature)?; + .map_err(|_| BubblegumOpsErrorKind::TryFromSignature)?; let lower = Signature::try_from(model.lower_bound_tx) - .map_err(|_| TreeErrorKind::TryFromSignature)?; + .map_err(|_| BubblegumOpsErrorKind::TryFromSignature)?; Ok(Self::new(tree, Some(upper), Some(lower))) } @@ -179,7 +158,7 @@ pub struct TreeHeaderResponse { } impl TryFrom for TreeHeaderResponse { - type Error = TreeErrorKind; + type Error = BubblegumOpsErrorKind; fn try_from(payload: ConcurrentMerkleTreeHeader) -> Result { let size = merkle_tree_get_size(&payload)?; @@ -227,7 +206,7 @@ impl TreeResponse { }) } - pub async fn all(client: &Rpc) -> Result, TreeErrorKind> { + pub async fn all(client: &Rpc) -> Result, BubblegumOpsErrorKind> { Ok(client .get_program_accounts( &id(), @@ -242,7 +221,10 @@ impl TreeResponse { .collect()) } - pub async fn find(client: &Rpc, pubkeys: Vec) -> Result, TreeErrorKind> { + pub async fn find( + client: &Rpc, + pubkeys: Vec, + ) -> Result, BubblegumOpsErrorKind> { let pubkeys: Vec = pubkeys .into_iter() .map(|p| Pubkey::from_str(&p)) @@ -258,7 +240,7 @@ impl TreeResponse { let results: Vec<(&Pubkey, Option)> = batch.iter().zip(accounts).collect(); - Ok::<_, TreeErrorKind>(results) + Ok::<_, BubblegumOpsErrorKind>(results) }) } @@ -271,7 +253,7 @@ impl TreeResponse { account.map(|account| Self::try_from_rpc(*pubkey, account)) }) .collect::, _>>() - .map_err(|_| TreeErrorKind::SerializeTreeResponse)?; + .map_err(|_| BubblegumOpsErrorKind::SerializeTreeResponse)?; Ok(trees) } From 0eced5d2bfa43f2c42890e79efaaf9ef276bb836 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 16 May 2024 14:36:59 +0200 Subject: [PATCH 26/72] refactor: use program transform in the account backfill no queuing into redis. --- Cargo.lock | 3 - ops/Cargo.toml | 3 - ops/src/account/account_details.rs | 30 --------- ops/src/account/account_info.rs | 28 +++++++++ ops/src/account/mod.rs | 2 +- ops/src/account/program.rs | 99 +++++++++++++++++++----------- ops/src/account/single.rs | 47 +++++--------- ops/src/bubblegum/README.md | 17 +---- 8 files changed, 108 insertions(+), 121 deletions(-) delete mode 100644 ops/src/account/account_details.rs create mode 100644 ops/src/account/account_info.rs diff --git a/Cargo.lock b/Cargo.lock index a2e20cfdc..f0ae67c19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1746,13 +1746,10 @@ dependencies = [ "digital_asset_types", "env_logger 0.10.0", "figment", - "flatbuffers", "futures", "indicatif", "log", "mpl-bubblegum", - "plerkle_messenger", - "plerkle_serialization", "program_transformers", "sea-orm", "solana-account-decoder", diff --git a/ops/Cargo.toml b/ops/Cargo.toml index 13cccc30b..48c0750e1 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -21,13 +21,10 @@ das-core = { workspace = true } digital_asset_types = { workspace = true } env_logger = { workspace = true } figment = { workspace = true } -flatbuffers = { workspace = true } futures = { workspace = true } indicatif = { workspace = true } log = { workspace = true } mpl-bubblegum = { workspace = true } -plerkle_messenger = { workspace = true } -plerkle_serialization = { workspace = true } program_transformers = { workspace = true } sea-orm = { workspace = true } solana-account-decoder = { workspace = true } diff --git a/ops/src/account/account_details.rs b/ops/src/account/account_details.rs deleted file mode 100644 index 0490f1ca1..000000000 --- a/ops/src/account/account_details.rs +++ /dev/null @@ -1,30 +0,0 @@ -use anyhow::Result; -use das_core::Rpc; -use solana_sdk::{account::Account, pubkey::Pubkey}; - -pub struct AccountDetails<'a> { - pub account: Account, - pub slot: u64, - pub pubkey: &'a Pubkey, -} - -impl<'a> AccountDetails<'a> { - pub fn new(account: Account, slot: u64, pubkey: &'a Pubkey) -> Self { - Self { - account, - slot, - pubkey, - } - } - - pub async fn fetch(rpc: &Rpc, pubkey: &'a Pubkey) -> Result { - let account_response = rpc.get_account(pubkey).await?; - let slot = account_response.context.slot; - - let account = account_response - .value - .ok_or_else(|| anyhow::anyhow!("Account not found for pubkey: {}", pubkey))?; - - Ok(Self::new(account, slot, pubkey)) - } -} diff --git a/ops/src/account/account_info.rs b/ops/src/account/account_info.rs new file mode 100644 index 000000000..b6216b331 --- /dev/null +++ b/ops/src/account/account_info.rs @@ -0,0 +1,28 @@ +use anyhow::Result; +use das_core::Rpc; +use program_transformers::AccountInfo; +use solana_sdk::pubkey::Pubkey; + +#[derive(thiserror::Error, Debug)] +pub enum AccountInfoError { + #[error("account not found for pubkey: {pubkey}")] + NotFound { pubkey: Pubkey }, + #[error("failed to fetch account info")] + SolanaRequestError(#[from] solana_client::client_error::ClientError), +} + +pub async fn fetch(rpc: &Rpc, pubkey: Pubkey) -> Result { + let account_response = rpc.get_account(&pubkey).await?; + let slot = account_response.context.slot; + + let account = account_response + .value + .ok_or_else(|| AccountInfoError::NotFound { pubkey })?; + + Ok(AccountInfo { + slot, + pubkey, + owner: account.owner, + data: account.data, + }) +} diff --git a/ops/src/account/mod.rs b/ops/src/account/mod.rs index 6eeff799f..59563b02e 100644 --- a/ops/src/account/mod.rs +++ b/ops/src/account/mod.rs @@ -1,4 +1,4 @@ -mod account_details; +mod account_info; mod cmd; mod program; mod single; diff --git a/ops/src/account/program.rs b/ops/src/account/program.rs index f3ff38736..76f78c2ff 100644 --- a/ops/src/account/program.rs +++ b/ops/src/account/program.rs @@ -1,20 +1,18 @@ +use super::account_info; use anyhow::Result; - -use super::account_details::AccountDetails; use clap::Parser; -use das_core::{MetricsArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs}; -use flatbuffers::FlatBufferBuilder; -use plerkle_serialization::{ - serializer::serialize_account, solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2, -}; +use das_core::{connect_db, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; +use futures::future::{ready, FutureExt}; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use program_transformers::{AccountInfo, ProgramTransformer}; use solana_sdk::pubkey::Pubkey; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::task; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Redis configuration - #[clap(flatten)] - pub queue: QueueArgs, - /// Metrics configuration #[clap(flatten)] pub metrics: MetricsArgs, @@ -30,6 +28,18 @@ pub struct Args { /// The public key of the program to backfill #[clap(value_parser = parse_pubkey)] pub program: Pubkey, + + /// The maximum buffer size for accounts + #[arg(long, env, default_value = "10000")] + pub max_buffer_size: usize, + + /// The number of worker threads + #[arg(long, env, default_value = "1000")] + pub account_worker_count: usize, + + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, } fn parse_pubkey(s: &str) -> Result { @@ -38,44 +48,59 @@ fn parse_pubkey(s: &str) -> Result { pub async fn run(config: Args) -> Result<()> { let rpc = Rpc::from_config(&config.solana); - let queue = QueuePool::try_from_config(&config.queue).await?; + let pool = connect_db(&config.database).await?; + let num_workers = config.account_worker_count; - let accounts = rpc.get_program_accounts(&config.program, None).await?; + let (tx, mut rx) = mpsc::channel::>(config.max_buffer_size); + + let mut workers = FuturesUnordered::new(); + let program_transformer = Arc::new(ProgramTransformer::new( + pool, + Box::new(|_info| ready(Ok(())).boxed()), + false, + )); + let account_info_worker_manager = tokio::spawn(async move { + while let Some(account_infos) = rx.recv().await { + if workers.len() >= num_workers { + workers.next().await; + } + + for account_info in account_infos { + let program_transformer = Arc::clone(&program_transformer); + + let worker = task::spawn(async move { + if let Err(e) = program_transformer + .handle_account_update(&account_info) + .await + { + error!("Failed to handle account update: {:?}", e); + } + }); + + workers.push(worker); + } + } + + while (workers.next().await).is_some() {} + }); + + let accounts = rpc.get_program_accounts(&config.program, None).await?; let accounts_chunks = accounts.chunks(config.batch_size); for batch in accounts_chunks { let results = futures::future::try_join_all( batch .iter() - .map(|(pubkey, _account)| AccountDetails::fetch(&rpc, pubkey)), + .cloned() + .map(|(pubkey, _account)| account_info::fetch(&rpc, pubkey)), ) .await?; - for account_detail in results { - let AccountDetails { - account, - slot, - pubkey, - } = account_detail; - let builder = FlatBufferBuilder::new(); - let account_info = ReplicaAccountInfoV2 { - pubkey: &pubkey.to_bytes(), - lamports: account.lamports, - owner: &account.owner.to_bytes(), - executable: account.executable, - rent_epoch: account.rent_epoch, - data: &account.data, - write_version: 0, - txn_signature: None, - }; - - let fbb = serialize_account(builder, &account_info, slot, false); - let bytes = fbb.finished_data(); - - queue.push_account_backfill(bytes).await?; - } + tx.send(results).await?; } + account_info_worker_manager.await?; + Ok(()) } diff --git a/ops/src/account/single.rs b/ops/src/account/single.rs index b40161fd2..a0482b862 100644 --- a/ops/src/account/single.rs +++ b/ops/src/account/single.rs @@ -1,19 +1,17 @@ use anyhow::Result; -use super::account_details::AccountDetails; +use super::account_info; use clap::Parser; -use das_core::{MetricsArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs}; -use flatbuffers::FlatBufferBuilder; -use plerkle_serialization::{ - serializer::serialize_account, solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2, -}; +use das_core::{connect_db, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; +use futures::future::{ready, FutureExt}; +use program_transformers::ProgramTransformer; use solana_sdk::pubkey::Pubkey; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Redis configuration + /// Database configuration #[clap(flatten)] - pub queue: QueueArgs, + pub database: PoolArgs, /// Metrics configuration #[clap(flatten)] @@ -33,29 +31,16 @@ fn parse_pubkey(s: &str) -> Result { pub async fn run(config: Args) -> Result<()> { let rpc = Rpc::from_config(&config.solana); - let queue = QueuePool::try_from_config(&config.queue).await?; - - let AccountDetails { - account, - slot, - pubkey, - } = AccountDetails::fetch(&rpc, &config.account).await?; - let builder = FlatBufferBuilder::new(); - let account_info = ReplicaAccountInfoV2 { - pubkey: &pubkey.to_bytes(), - lamports: account.lamports, - owner: &account.owner.to_bytes(), - executable: account.executable, - rent_epoch: account.rent_epoch, - data: &account.data, - write_version: 0, - txn_signature: None, - }; - - let fbb = serialize_account(builder, &account_info, slot, false); - let bytes = fbb.finished_data(); - - queue.push_account_backfill(bytes).await?; + let pool = connect_db(&config.database).await?; + + let program_transformer = + ProgramTransformer::new(pool, Box::new(|_info| ready(Ok(())).boxed()), false); + + let account_info = account_info::fetch(&rpc, config.account).await?; + + program_transformer + .handle_account_update(&account_info) + .await?; Ok(()) } diff --git a/ops/src/bubblegum/README.md b/ops/src/bubblegum/README.md index d1dc5772b..397dbb234 100644 --- a/ops/src/bubblegum/README.md +++ b/ops/src/bubblegum/README.md @@ -8,25 +8,10 @@ Command line arguments can also be set through environment variables. ### Backfill -The `backfill` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL, and the messenger Redis URL. +The `backfill` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL. **warning**: The command expects full archive access to transactions. Before proceeding ensure your RPC is able to serve complete transaction history for Solana. -```mermaid -flowchart - start((Start)) -->init[Initialize RPC, DB] - init --> fetchTreesDB[Fetch Trees from DB] - fetchTreesDB --> findGapsDB[Find Gaps in DB] - findGapsDB --> enqueueGapFills[Enqueue Gap Fills] - enqueueGapFills --> gapWorkerManager[Gap Worker Manager] - gapWorkerManager --> crawlSignatures[Crawl Solana RPC] - crawlSignatures --> enqueueSignatures[Enqueue Signatures] - enqueueSignatures --> transactionWorkerManager[Transaction Worker Manager] - transactionWorkerManager --> fetchTransactionsRPC[Fetch Transactions RPC] - fetchTransactionsRPC --> processTransactions[Push Transaction to Messenger] - processTransactions ---> Finished -``` - ``` Usage: das-ops bubblegum backfill [OPTIONS] --database-url --messenger-redis-url --solana-rpc-url From 138bf2ed5151f05c61b7c3d83f9a7b625b85a950 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 30 May 2024 14:15:55 +0200 Subject: [PATCH 27/72] refactor: move bubble backfill to a lib so can be used by other projects eg LightDAS and be included in the grpc-ingest to execute backfills on restart then pick up tasks from the stream to insure no out of order indexing --- Builder.Dockerfile | 7 +- Cargo.lock | 67 +++- Cargo.toml | 27 +- Proxy.Dockerfile | 1 + backfill/Cargo.toml | 40 ++ backfill/README.md | 24 ++ backfill/src/error.rs | 19 + backfill/src/gap.rs | 140 +++++++ backfill/src/lib.rs | 69 ++++ backfill/src/tree.rs | 118 ++++++ backfill/src/worker/gap.rs | 65 +++ backfill/src/worker/mod.rs | 9 + backfill/src/worker/program_transformer.rs | 49 +++ backfill/src/worker/transaction.rs | 210 ++++++++++ backfill/src/worker/tree.rs | 116 ++++++ core/Cargo.toml | 28 +- core/src/lib.rs | 2 + core/src/metadata_json.rs | 223 +++++++++++ nft_ingester/Cargo.toml | 1 + nft_ingester/src/tasks/common/mod.rs | 2 +- ops/Cargo.toml | 2 +- ops/src/account/account_info.rs | 2 +- ops/src/account/program.rs | 32 +- ops/src/account/single.rs | 32 +- ops/src/bubblegum/README.md | 60 +-- ops/src/bubblegum/audit.rs | 89 ----- ops/src/bubblegum/backfiller.rs | 443 +-------------------- ops/src/bubblegum/cmd.rs | 29 +- ops/src/bubblegum/mod.rs | 2 - ops/src/bubblegum/tree.rs | 260 ------------ program_transformers/Cargo.toml | 6 +- program_transformers/src/lib.rs | 29 +- 32 files changed, 1261 insertions(+), 942 deletions(-) create mode 100644 backfill/Cargo.toml create mode 100644 backfill/README.md create mode 100644 backfill/src/error.rs create mode 100644 backfill/src/gap.rs create mode 100644 backfill/src/lib.rs create mode 100644 backfill/src/tree.rs create mode 100644 backfill/src/worker/gap.rs create mode 100644 backfill/src/worker/mod.rs create mode 100644 backfill/src/worker/program_transformer.rs create mode 100644 backfill/src/worker/transaction.rs create mode 100644 backfill/src/worker/tree.rs create mode 100644 core/src/metadata_json.rs delete mode 100644 ops/src/bubblegum/audit.rs delete mode 100644 ops/src/bubblegum/tree.rs diff --git a/Builder.Dockerfile b/Builder.Dockerfile index 4ea85b7c9..6ee7cbaed 100644 --- a/Builder.Dockerfile +++ b/Builder.Dockerfile @@ -1,11 +1,12 @@ FROM rust:1.75-bullseye AS builder RUN apt-get update -y && \ - apt-get install -y build-essential make git - + apt-get install -y build-essential make git + RUN mkdir /rust RUN mkdir /rust/bins COPY Cargo.toml /rust COPY core /rust/core +COPY backfill /rust/backfill COPY das_api /rust/das_api COPY digital_asset_types /rust/digital_asset_types COPY integration_tests /rust/integration_tests @@ -19,7 +20,7 @@ COPY blockbuster rust/blockbuster WORKDIR /rust RUN --mount=type=cache,target=/rust/target,id=das-rust \ cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins - + FROM rust:1.75-slim-bullseye as final COPY --from=builder /rust/bins /das/ CMD echo "Built the DAS API bins!" diff --git a/Cargo.lock b/Cargo.lock index f0ae67c19..ff9e1e45f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,6 +1524,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation" version = "0.9.3" @@ -1710,24 +1716,68 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "das-backfill" +version = "0.7.2" +dependencies = [ + "anchor-client", + "anyhow", + "blockbuster", + "borsh 0.10.3", + "bs58 0.4.0", + "clap 4.4.8", + "das-core", + "digital_asset_types", + "futures", + "heck 0.5.0", + "log", + "mpl-bubblegum", + "num-traits", + "program_transformers", + "sea-orm", + "serde_json", + "solana-client", + "solana-program", + "solana-sdk", + "solana-transaction-status", + "spl-account-compression", + "spl-token", + "sqlx", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "das-core" version = "0.7.2" dependencies = [ "anyhow", "backon", + "borsh 0.10.3", + "bs58 0.4.0", "cadence", "cadence-macros", "clap 4.4.8", + "derive_more", + "digital_asset_types", "figment", + "futures", + "indicatif", + "log", "plerkle_messenger", + "reqwest", + "sea-orm", + "serde_json", "solana-account-decoder", "solana-client", "solana-sdk", "solana-transaction-status", + "spl-account-compression", "sqlx", "thiserror", "tokio", + "url", ] [[package]] @@ -1736,12 +1786,12 @@ version = "0.7.2" dependencies = [ "anchor-client", "anyhow", - "backon", "borsh 0.10.3", "bs58 0.4.0", "cadence", "cadence-macros", "clap 4.4.8", + "das-backfill", "das-core", "digital_asset_types", "env_logger 0.10.0", @@ -1877,6 +1927,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + [[package]] name = "deunicode" version = "1.4.3" @@ -3544,6 +3607,7 @@ dependencies = [ "cadence-macros", "chrono", "clap 4.4.8", + "das-core", "digital_asset_types", "figment", "flatbuffers", @@ -4276,6 +4340,7 @@ dependencies = [ "bs58 0.4.0", "cadence", "cadence-macros", + "das-core", "digital_asset_types", "futures", "heck 0.5.0", diff --git a/Cargo.toml b/Cargo.toml index 4d0b859c9..72e189cc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "backfill", "blockbuster", "core", "das_api", @@ -32,18 +33,20 @@ anyhow = "1.0.75" async-std = "1.0.0" async-trait = "0.1.60" backon = "0.4.1" -blockbuster = {path = "blockbuster"} +blockbuster = { path = "blockbuster" } borsh = "~0.10.3" borsh-derive = "~0.10.3" bs58 = "0.4.0" -bytemuck = {version = "1.14.0", features = ["derive"]} +bytemuck = { version = "1.14.0", features = ["derive"] } cadence = "0.29.0" cadence-macros = "0.29.0" chrono = "0.4.19" clap = "4.2.2" -das-core = {path = "core"} -das_api = {path = "das_api"} -digital_asset_types = {path = "digital_asset_types"} +das-core = { path = "core" } +das-backfill = { path = "backfill" } +das_api = { path = "das_api" } +derive_more = { version = "0.99.17" } +digital_asset_types = { path = "digital_asset_types" } enum-iterator = "1.2.0" enum-iterator-derive = "1.1.0" env_logger = "0.10.0" @@ -65,12 +68,12 @@ jsonrpsee-core = "0.16.2" lazy_static = "1.4.0" log = "0.4.17" metrics = "0.20.1" -migration = {path = "migration"} +migration = { path = "migration" } mime_guess = "2.0.4" mpl-bubblegum = "1.2.0" -mpl-core = {version = "0.7.1", features = ["serde"]} +mpl-core = { version = "0.7.1", features = ["serde"] } mpl-token-metadata = "4.1.1" -nft_ingester = {path = "nft_ingester"} +nft_ingester = { path = "nft_ingester" } num-derive = "0.3.3" num-traits = "0.2.15" once_cell = "1.19.0" @@ -78,7 +81,7 @@ open-rpc-derive = "0.0.4" open-rpc-schema = "0.0.4" plerkle_messenger = "1.6.0" plerkle_serialization = "1.8.0" -program_transformers = {path = "program_transformers"} +program_transformers = { path = "program_transformers" } prometheus = "0.13.3" proxy-wasm = "0.2.0" rand = "0.8.5" @@ -104,9 +107,9 @@ spl-account-compression = "0.3.0" spl-associated-token-account = ">= 1.1.3, < 3.0" spl-concurrent-merkle-tree = "0.2.0" spl-noop = "0.2.0" -spl-pod = {version = "0.1.0", features = ["serde-traits"]} +spl-pod = { version = "0.1.0", features = ["serde-traits"] } spl-token = ">= 3.5.0, < 5.0" -spl-token-2022 = {version = "1.0", features = ["no-entrypoint"]} +spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] } spl-token-group-interface = "0.1.0" spl-token-metadata-interface = "0.2.0" sqlx = "0.6.2" @@ -118,7 +121,7 @@ tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" tracing-subscriber = "0.3.16" -txn_forwarder = {path = "tools/txn_forwarder"} +txn_forwarder = { path = "tools/txn_forwarder" } url = "2.3.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" diff --git a/Proxy.Dockerfile b/Proxy.Dockerfile index 9679a632e..add959fab 100644 --- a/Proxy.Dockerfile +++ b/Proxy.Dockerfile @@ -4,6 +4,7 @@ RUN cargo install wasm-pack RUN mkdir /rust COPY ./Cargo.toml /rust COPY ./core /rust/core +COPY ./backfill /rust/backfill COPY ./das_api /rust/das_api COPY ./digital_asset_types /rust/digital_asset_types COPY ./integration_tests /rust/integration_tests diff --git a/backfill/Cargo.toml b/backfill/Cargo.toml new file mode 100644 index 000000000..ce082f81c --- /dev/null +++ b/backfill/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "das-backfill" +version = { workspace = true } +edition = { workspace = true } +repository = { workspace = true } +publish = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +blockbuster = { workspace = true } +bs58 = { workspace = true } +das-core = { workspace = true } +solana-client = { workspace = true } +borsh = { workspace = true } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } +anchor-client = { workspace = true } +futures = { workspace = true } +clap = { workspace = true } +log = { workspace = true } +solana-program = { workspace = true } +program_transformers = { workspace = true } +heck = { workspace = true } +mpl-bubblegum = { workspace = true } +num-traits = { workspace = true } +sea-orm = { workspace = true } +serde_json = { workspace = true } +solana-sdk = { workspace = true } +solana-transaction-status = { workspace = true } +spl-account-compression = { workspace = true, features = ["no-entrypoint"] } +spl-token = { workspace = true, features = ["no-entrypoint"] } +sqlx = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["time"] } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/backfill/README.md b/backfill/README.md new file mode 100644 index 000000000..e98983303 --- /dev/null +++ b/backfill/README.md @@ -0,0 +1,24 @@ +## DAS Backfill + +The DAS Backfill library facilitates the initial setup and data backfilling for DAS, focusing on the bubblegum program. This program's indexing heavily relies on transaction data. While the library supports parallel backfilling across different trees, it ensures that transactions within each tree are processed sequentially. This approach guarantees accurate representation of every modification in the merkle tree within DAS. + +## Usage + +```rust +use das_backfill::{ + BubblegumBackfillArgs, + BubblegumBackfillContext, + start_bubblegum_backfill +}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let database_pool = sqlx::PgPool::connect("your_database_url").await?; + let solana_rpc = Rpc::new("your_solana_rpc_url"); + + let context = BubblegumBackfillContext::new(database_pool, solana_rpc); + let args = BubblegumBackfillArgs::parse(); // Parses args from CLI + + start_bubblegum_backfill(context, args).await +} +``` diff --git a/backfill/src/error.rs b/backfill/src/error.rs new file mode 100644 index 000000000..420a15a52 --- /dev/null +++ b/backfill/src/error.rs @@ -0,0 +1,19 @@ +#[derive(Debug, thiserror::Error)] +pub enum ErrorKind { + #[error("anchor")] + Anchor(#[from] anchor_client::anchor_lang::error::Error), + #[error("solana rpc")] + Rpc(#[from] solana_client::client_error::ClientError), + #[error("parse pubkey")] + ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), + #[error("serialize tree response")] + SerializeTreeResponse, + #[error("sea orm")] + Database(#[from] sea_orm::DbErr), + #[error("try from pubkey")] + TryFromPubkey, + #[error("try from signature")] + TryFromSignature, + #[error("generic error: {0}")] + Generic(String), +} diff --git a/backfill/src/gap.rs b/backfill/src/gap.rs new file mode 100644 index 000000000..feb523b98 --- /dev/null +++ b/backfill/src/gap.rs @@ -0,0 +1,140 @@ +use super::ErrorKind; +use crate::Rpc; +use anyhow::Result; +use clap::Args; +use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; +use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use std::str::FromStr; +use tokio::sync::mpsc::Sender; + +const GET_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1000; + +#[derive(Debug, Clone, Args)] +pub struct ConfigBackfiller { + /// Solana RPC URL + #[arg(long, env)] + pub solana_rpc_url: String, +} + +const TREE_GAP_SQL: &str = r#" +WITH sequenced_data AS ( + SELECT + tree, + seq, + LEAD(seq) OVER (ORDER BY seq ASC) AS next_seq, + tx AS current_tx, + LEAD(tx) OVER (ORDER BY seq ASC) AS next_tx + FROM + cl_audits_v2 + WHERE + tree = $1 +), +gaps AS ( + SELECT + tree, + seq AS gap_start_seq, + next_seq AS gap_end_seq, + current_tx AS lower_bound_tx, + next_tx AS upper_bound_tx + FROM + sequenced_data + WHERE + next_seq IS NOT NULL AND + next_seq - seq > 1 +) +SELECT + tree, + gap_start_seq, + gap_end_seq, + lower_bound_tx, + upper_bound_tx +FROM + gaps +ORDER BY + gap_start_seq; +"#; + +#[derive(Debug, FromQueryResult, PartialEq, Clone)] +pub struct TreeGapModel { + pub tree: Vec, + pub gap_start_seq: i64, + pub gap_end_seq: i64, + pub lower_bound_tx: Vec, + pub upper_bound_tx: Vec, +} + +impl TreeGapModel { + pub async fn find(conn: &DatabaseConnection, tree: Pubkey) -> Result, ErrorKind> { + let statement = Statement::from_sql_and_values( + DbBackend::Postgres, + TREE_GAP_SQL, + vec![Value::Bytes(Some(Box::new(tree.as_ref().to_vec())))], + ); + + TreeGapModel::find_by_statement(statement) + .all(conn) + .await + .map_err(Into::into) + } +} + +impl TryFrom for TreeGapFill { + type Error = ErrorKind; + + fn try_from(model: TreeGapModel) -> Result { + let tree = Pubkey::try_from(model.tree).map_err(|_| ErrorKind::TryFromPubkey)?; + let upper = + Signature::try_from(model.upper_bound_tx).map_err(|_| ErrorKind::TryFromSignature)?; + let lower = + Signature::try_from(model.lower_bound_tx).map_err(|_| ErrorKind::TryFromSignature)?; + + Ok(Self::new(tree, Some(upper), Some(lower))) + } +} + +pub struct TreeGapFill { + tree: Pubkey, + before: Option, + until: Option, +} + +impl TreeGapFill { + pub const fn new(tree: Pubkey, before: Option, until: Option) -> Self { + Self { + tree, + before, + until, + } + } + + pub async fn crawl(&self, client: Rpc, sender: Sender) -> Result<()> { + let mut before = self.before; + + loop { + let sigs = client + .get_signatures_for_address(&self.tree, before, self.until) + .await?; + let sig_count = sigs.len(); + + let successful_transactions = sigs + .into_iter() + .filter(|transaction| transaction.err.is_none()) + .collect::>(); + + for sig in successful_transactions.iter() { + let sig = Signature::from_str(&sig.signature)?; + + sender.send(sig).await?; + + before = Some(sig); + } + + if sig_count < GET_SIGNATURES_FOR_ADDRESS_LIMIT { + break; + } + } + + Ok(()) + } +} diff --git a/backfill/src/lib.rs b/backfill/src/lib.rs new file mode 100644 index 000000000..c6973f85b --- /dev/null +++ b/backfill/src/lib.rs @@ -0,0 +1,69 @@ +mod error; +mod gap; +mod tree; +mod worker; + +pub use error::ErrorKind; + +use anyhow::Result; +use clap::Parser; +use das_core::Rpc; +use futures::{stream::FuturesUnordered, StreamExt}; +use tree::TreeResponse; +use worker::TreeWorkerArgs; + +#[derive(Clone)] +pub struct BubblegumBackfillContext { + pub database_pool: sqlx::PgPool, + pub solana_rpc: Rpc, +} + +impl BubblegumBackfillContext { + pub const fn new(database_pool: sqlx::PgPool, solana_rpc: Rpc) -> Self { + Self { + database_pool, + solana_rpc, + } + } +} + +#[derive(Debug, Parser, Clone)] +pub struct BubblegumBackfillArgs { + /// Number of tree crawler workers + #[arg(long, env, default_value = "20")] + pub tree_crawler_count: usize, + + /// The list of trees to crawl. If not specified, all trees will be crawled. + #[arg(long, env, use_value_delimiter = true)] + pub only_trees: Option>, + + #[clap(flatten)] + pub tree_worker: TreeWorkerArgs, +} + +pub async fn start_bubblegum_backfill( + context: BubblegumBackfillContext, + args: BubblegumBackfillArgs, +) -> Result<()> { + let trees = if let Some(ref only_trees) = args.only_trees { + TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? + } else { + TreeResponse::all(&context.solana_rpc).await? + }; + + let mut crawl_handles = FuturesUnordered::new(); + + for tree in trees { + if crawl_handles.len() >= args.tree_crawler_count { + crawl_handles.next().await; + } + let context = context.clone(); + let handle = args.tree_worker.start(context, tree); + + crawl_handles.push(handle); + } + + futures::future::try_join_all(crawl_handles).await?; + + Ok(()) +} diff --git a/backfill/src/tree.rs b/backfill/src/tree.rs new file mode 100644 index 000000000..efb5c1cbf --- /dev/null +++ b/backfill/src/tree.rs @@ -0,0 +1,118 @@ +use super::ErrorKind; +use anyhow::Result; +use borsh::BorshDeserialize; +use das_core::Rpc; +use solana_client::rpc_filter::{Memcmp, RpcFilterType}; +use solana_sdk::{account::Account, pubkey::Pubkey}; +use spl_account_compression::id; +use spl_account_compression::state::{ + merkle_tree_get_size, ConcurrentMerkleTreeHeader, CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1, +}; +use std::str::FromStr; + +#[derive(Debug, Clone)] +pub struct TreeHeaderResponse { + pub max_depth: u32, + pub max_buffer_size: u32, + pub creation_slot: u64, + pub size: usize, +} + +impl TryFrom for TreeHeaderResponse { + type Error = ErrorKind; + + fn try_from(payload: ConcurrentMerkleTreeHeader) -> Result { + let size = merkle_tree_get_size(&payload)?; + + Ok(Self { + max_depth: payload.get_max_depth(), + max_buffer_size: payload.get_max_buffer_size(), + creation_slot: payload.get_creation_slot(), + size, + }) + } +} + +#[derive(Debug, Clone)] +pub struct TreeResponse { + pub pubkey: Pubkey, + pub tree_header: TreeHeaderResponse, + pub seq: u64, +} + +impl TreeResponse { + pub fn try_from_rpc(pubkey: Pubkey, account: Account) -> Result { + let bytes = account.data.as_slice(); + + let (header_bytes, rest) = bytes.split_at(CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1); + let header: ConcurrentMerkleTreeHeader = + ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?; + + let merkle_tree_size = merkle_tree_get_size(&header)?; + let (tree_bytes, _canopy_bytes) = rest.split_at(merkle_tree_size); + + let seq_bytes = tree_bytes[0..8].try_into()?; + let seq = u64::from_le_bytes(seq_bytes); + + let (auth, _) = Pubkey::find_program_address(&[pubkey.as_ref()], &mpl_bubblegum::ID); + + header.assert_valid_authority(&auth)?; + + let tree_header = header.try_into()?; + + Ok(Self { + pubkey, + tree_header, + seq, + }) + } + + pub async fn all(client: &Rpc) -> Result, ErrorKind> { + Ok(client + .get_program_accounts( + &id(), + Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes( + 0, + vec![1u8], + ))]), + ) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Self::try_from_rpc(pubkey, account).ok()) + .collect()) + } + + pub async fn find(client: &Rpc, pubkeys: Vec) -> Result, ErrorKind> { + let pubkeys: Vec = pubkeys + .into_iter() + .map(|p| Pubkey::from_str(&p)) + .collect::, _>>()?; + let pubkey_batches = pubkeys.chunks(100); + let pubkey_batches_count = pubkey_batches.len(); + + let mut gma_handles = Vec::with_capacity(pubkey_batches_count); + + for batch in pubkey_batches { + gma_handles.push(async move { + let accounts = client.get_multiple_accounts(batch).await?; + + let results: Vec<(&Pubkey, Option)> = batch.iter().zip(accounts).collect(); + + Ok::<_, ErrorKind>(results) + }) + } + + let result = futures::future::try_join_all(gma_handles).await?; + + let trees = result + .into_iter() + .flatten() + .filter_map(|(pubkey, account)| { + account.map(|account| Self::try_from_rpc(*pubkey, account)) + }) + .collect::, _>>() + .map_err(|_| ErrorKind::SerializeTreeResponse)?; + + Ok(trees) + } +} diff --git a/backfill/src/worker/gap.rs b/backfill/src/worker/gap.rs new file mode 100644 index 000000000..07b88f4f5 --- /dev/null +++ b/backfill/src/worker/gap.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use clap::Parser; +use das_core::Rpc; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use solana_sdk::signature::Signature; +use tokio::{ + sync::mpsc::{channel, Sender}, + task::JoinHandle, +}; + +use crate::gap::TreeGapFill; +use crate::BubblegumBackfillContext; + +#[derive(Parser, Debug, Clone)] +pub struct GapWorkerArgs { + /// The size of the signature channel. + #[arg(long, env, default_value = "1000")] + pub gap_channel_size: usize, + + /// The number of gap workers. + #[arg(long, env, default_value = "25")] + pub gap_worker_count: usize, +} + +impl GapWorkerArgs { + pub fn start( + &self, + context: BubblegumBackfillContext, + forward: Sender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (gap_sender, mut gap_receiver) = channel::(self.gap_channel_size); + let gap_worker_count = self.gap_worker_count; + + let handler = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + let sender = forward.clone(); + + while let Some(gap) = gap_receiver.recv().await { + if handlers.len() >= gap_worker_count { + handlers.next().await; + } + + let client = context.solana_rpc.clone(); + let sender = sender.clone(); + + let handle = spawn_crawl_worker(client, sender, gap); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + }); + + Ok((handler, gap_sender)) + } +} + +fn spawn_crawl_worker(client: Rpc, sender: Sender, gap: TreeGapFill) -> JoinHandle<()> { + tokio::spawn(async move { + if let Err(e) = gap.crawl(client, sender).await { + error!("tree transaction: {:?}", e); + } + }) +} diff --git a/backfill/src/worker/mod.rs b/backfill/src/worker/mod.rs new file mode 100644 index 000000000..0bc7f9e36 --- /dev/null +++ b/backfill/src/worker/mod.rs @@ -0,0 +1,9 @@ +mod gap; +mod program_transformer; +mod transaction; +mod tree; + +pub use gap::GapWorkerArgs; +pub use program_transformer::ProgramTransformerWorkerArgs; +pub use transaction::SignatureWorkerArgs; +pub use tree::TreeWorkerArgs; diff --git a/backfill/src/worker/program_transformer.rs b/backfill/src/worker/program_transformer.rs new file mode 100644 index 000000000..1fa487b49 --- /dev/null +++ b/backfill/src/worker/program_transformer.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use clap::Parser; +use das_core::{create_download_metadata_notifier, DownloadMetadataInfo}; +use log::error; +use program_transformers::{ProgramTransformer, TransactionInfo}; +use tokio::sync::mpsc::{channel, Sender, UnboundedSender}; +use tokio::task::JoinHandle; + +use crate::BubblegumBackfillContext; + +#[derive(Parser, Debug, Clone)] +pub struct ProgramTransformerWorkerArgs { + #[arg(long, env, default_value = "100000")] + pub program_transformer_channel_size: usize, +} + +impl ProgramTransformerWorkerArgs { + pub fn start( + &self, + context: BubblegumBackfillContext, + forwarder: UnboundedSender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (sender, mut receiver) = + channel::(self.program_transformer_channel_size); + + let handle = tokio::spawn(async move { + let mut transactions = Vec::new(); + let pool = context.database_pool.clone(); + + let download_metadata_notifier = create_download_metadata_notifier(forwarder).await; + + let program_transformer = ProgramTransformer::new(pool, download_metadata_notifier); + + while let Some(gap) = receiver.recv().await { + transactions.push(gap); + } + + transactions.sort_by(|a, b| b.signature.cmp(&a.signature)); + + for transaction in transactions { + if let Err(e) = program_transformer.handle_transaction(&transaction).await { + error!("handle transaction: {:?}", e) + }; + } + }); + + Ok((handle, sender)) + } +} diff --git a/backfill/src/worker/transaction.rs b/backfill/src/worker/transaction.rs new file mode 100644 index 000000000..c047cffe4 --- /dev/null +++ b/backfill/src/worker/transaction.rs @@ -0,0 +1,210 @@ +use crate::error::ErrorKind; +use anyhow::Result; +use clap::Parser; +use das_core::Rpc; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use program_transformers::TransactionInfo; +use solana_program::pubkey::Pubkey; +use solana_sdk::instruction::CompiledInstruction; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::VersionedTransaction; +use solana_transaction_status::{ + option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta, + InnerInstruction, InnerInstructions, UiInstruction, +}; +use tokio::{ + sync::mpsc::{channel, Sender}, + task::JoinHandle, +}; + +pub struct PubkeyString(pub String); + +impl TryFrom for Pubkey { + type Error = ErrorKind; + + fn try_from(value: PubkeyString) -> Result { + let decoded_bytes = bs58::decode(value.0) + .into_vec() + .map_err(|e| ErrorKind::Generic(e.to_string()))?; + + Pubkey::try_from(decoded_bytes) + .map_err(|_| ErrorKind::Generic("unable to convert pubkey".to_string())) + } +} + +pub struct FetchedEncodedTransactionWithStatusMeta(pub EncodedConfirmedTransactionWithStatusMeta); + +impl TryFrom for TransactionInfo { + type Error = ErrorKind; + + fn try_from( + fetched_transaction: FetchedEncodedTransactionWithStatusMeta, + ) -> Result { + let mut account_keys = Vec::new(); + let encoded_transaction_with_status_meta = fetched_transaction.0; + + let ui_transaction: VersionedTransaction = encoded_transaction_with_status_meta + .transaction + .transaction + .decode() + .ok_or(ErrorKind::Generic( + "unable to decode transaction".to_string(), + ))?; + + let signature = ui_transaction.signatures[0]; + + let msg = ui_transaction.message; + + let meta = encoded_transaction_with_status_meta + .transaction + .meta + .ok_or(ErrorKind::Generic( + "unable to get meta from transaction".to_string(), + ))?; + + for address in msg.static_account_keys().iter().copied() { + account_keys.push(address); + } + let ui_loaded_addresses = meta.loaded_addresses; + + let message_address_table_lookup = msg.address_table_lookups(); + + if message_address_table_lookup.is_some() { + if let OptionSerializer::Some(ui_lookup_table) = ui_loaded_addresses { + for address in ui_lookup_table.writable { + account_keys.push(PubkeyString(address).try_into()?); + } + + for address in ui_lookup_table.readonly { + account_keys.push(PubkeyString(address).try_into()?); + } + } + } + + let mut meta_inner_instructions = Vec::new(); + + let compiled_instruction = msg.instructions().to_vec(); + + let mut instructions = Vec::new(); + + for inner in compiled_instruction { + instructions.push(InnerInstruction { + stack_height: Some(0), + instruction: CompiledInstruction { + program_id_index: inner.program_id_index, + accounts: inner.accounts, + data: inner.data, + }, + }); + } + + meta_inner_instructions.push(InnerInstructions { + index: 0, + instructions, + }); + + if let OptionSerializer::Some(inner_instructions) = meta.inner_instructions { + for ix in inner_instructions { + let mut instructions = Vec::new(); + + for inner in ix.instructions { + if let UiInstruction::Compiled(compiled) = inner { + instructions.push(InnerInstruction { + stack_height: compiled.stack_height, + instruction: CompiledInstruction { + program_id_index: compiled.program_id_index, + accounts: compiled.accounts, + data: bs58::decode(compiled.data) + .into_vec() + .map_err(|e| ErrorKind::Generic(e.to_string()))?, + }, + }); + } + } + + meta_inner_instructions.push(InnerInstructions { + index: ix.index, + instructions, + }); + } + } + + Ok(Self { + slot: encoded_transaction_with_status_meta.slot, + account_keys, + signature, + message_instructions: msg.instructions().to_vec(), + meta_inner_instructions, + }) + } +} + +#[derive(Parser, Clone, Debug)] +pub struct SignatureWorkerArgs { + /// The size of the signature channel. + #[arg(long, env, default_value = "100000")] + pub signature_channel_size: usize, + /// The number of transaction workers. + #[arg(long, env, default_value = "50")] + pub signature_worker_count: usize, +} + +impl SignatureWorkerArgs { + pub fn start( + &self, + context: crate::BubblegumBackfillContext, + forwarder: Sender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (sig_sender, mut sig_receiver) = channel::(self.signature_channel_size); + let worker_count = self.signature_worker_count; + + let handle = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + + while let Some(signature) = sig_receiver.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } + + let solana_rpc = context.solana_rpc.clone(); + let transaction_sender = forwarder.clone(); + + let handle = spawn_transaction_worker(solana_rpc, transaction_sender, signature); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + }); + + Ok((handle, sig_sender)) + } +} + +async fn queue_transaction<'a>( + client: Rpc, + sender: Sender, + signature: Signature, +) -> Result<(), ErrorKind> { + let transaction = client.get_transaction(&signature).await?; + + sender + .send(FetchedEncodedTransactionWithStatusMeta(transaction).try_into()?) + .await + .map_err(|e| ErrorKind::Generic(e.to_string()))?; + + Ok(()) +} + +fn spawn_transaction_worker( + client: Rpc, + sender: Sender, + signature: Signature, +) -> JoinHandle<()> { + tokio::spawn(async move { + if let Err(e) = queue_transaction(client, sender, signature).await { + error!("queue transaction: {:?}", e); + } + }) +} diff --git a/backfill/src/worker/tree.rs b/backfill/src/worker/tree.rs new file mode 100644 index 000000000..e75917e52 --- /dev/null +++ b/backfill/src/worker/tree.rs @@ -0,0 +1,116 @@ +use crate::{ + gap::{TreeGapFill, TreeGapModel}, + tree::TreeResponse, + BubblegumBackfillContext, +}; +use anyhow::Result; +use clap::Parser; +use das_core::MetadataJsonDownloadWorkerArgs; +use digital_asset_types::dao::cl_audits_v2; +use log::error; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector}; +use solana_sdk::signature::Signature; +use tokio::task::JoinHandle; + +use super::{GapWorkerArgs, ProgramTransformerWorkerArgs, SignatureWorkerArgs}; + +#[derive(Debug, Clone, Parser)] +pub struct TreeWorkerArgs { + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + + #[clap(flatten)] + pub signature_worker: SignatureWorkerArgs, + + #[clap(flatten)] + pub gap_worker: GapWorkerArgs, + + #[clap(flatten)] + pub program_transformer_worker: ProgramTransformerWorkerArgs, +} +impl TreeWorkerArgs { + pub fn start( + &self, + context: BubblegumBackfillContext, + tree: TreeResponse, + ) -> JoinHandle> { + let db_pool = context.database_pool.clone(); + let metadata_json_download_db_pool = context.database_pool.clone(); + + let program_transformer_context = context.clone(); + let signature_context = context.clone(); + + let metadata_json_download_worker_args = self.metadata_json_download_worker.clone(); + let program_transformer_worker_args = self.program_transformer_worker.clone(); + let signature_worker_args = self.signature_worker.clone(); + let gap_worker_args = self.gap_worker.clone(); + + tokio::spawn(async move { + let (metadata_json_download_worker, metadata_json_download_sender) = + metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; + + let (program_transformer_worker, transaction_info_sender) = + program_transformer_worker_args + .start(program_transformer_context, metadata_json_download_sender)?; + + let (signature_worker, signature_sender) = + signature_worker_args.start(signature_context, transaction_info_sender)?; + + let (gap_worker, tree_gap_sender) = gap_worker_args.start(context, signature_sender)?; + + { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(db_pool); + + let mut gaps = TreeGapModel::find(&conn, tree.pubkey) + .await? + .into_iter() + .map(TryInto::try_into) + .collect::, _>>()?; + + let upper_known_seq = cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_desc(cl_audits_v2::Column::Seq) + .one(&conn) + .await?; + + let lower_known_seq = cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_asc(cl_audits_v2::Column::Seq) + .one(&conn) + .await?; + + if let Some(upper_seq) = upper_known_seq { + let signature = Signature::try_from(upper_seq.tx.as_ref())?; + + gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); + } else if tree.seq > 0 { + gaps.push(TreeGapFill::new(tree.pubkey, None, None)); + } + + if let Some(lower_seq) = lower_known_seq.filter(|seq| seq.seq > 1) { + let signature = Signature::try_from(lower_seq.tx.as_ref())?; + + gaps.push(TreeGapFill::new(tree.pubkey, Some(signature), None)); + } + + for gap in gaps { + if let Err(e) = tree_gap_sender.send(gap).await { + error!("send gap: {:?}", e); + } + } + } + + drop(tree_gap_sender); + + futures::future::try_join4( + gap_worker, + signature_worker, + program_transformer_worker, + metadata_json_download_worker, + ) + .await?; + + Ok(()) + }) + } +} diff --git a/core/Cargo.toml b/core/Cargo.toml index d0532684f..9ee91c67f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -8,18 +8,34 @@ publish.workspace = true [dependencies] anyhow = { workspace = true } backon = { workspace = true } +borsh = { workspace = true } +bs58 = { workspace = true } +cadence = { workspace = true } +cadence-macros = { workspace = true } +clap = { workspace = true, features = ["derive", "cargo", "env"] } +derive_more = { workspace = true } +digital_asset_types = { workspace = true } +figment = { workspace = true } +futures = { workspace = true } +indicatif = { workspace = true } +log = { workspace = true } +plerkle_messenger = { workspace = true } +reqwest = { workspace = true } +sea-orm = { workspace = true, features = [ + "sqlx-postgres", + "with-chrono", + "runtime-tokio-rustls", +] } +serde_json = { workspace = true } +spl-account-compression = { workspace = true } solana-account-decoder = { workspace = true } solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } -cadence = { workspace = true } -cadence-macros = { workspace = true } +sqlx = { workspace = true, fatures = ["runtime-tokio-rustls", "postgres"] } thiserror = { workspace = true } -figment = { workspace = true } -plerkle_messenger = { workspace = true } tokio = { workspace = true } -clap = { workspace = true, features = ["derive", "cargo", "env"] } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } +url = { workspace = true } [lints] workspace = true diff --git a/core/src/lib.rs b/core/src/lib.rs index da6bb050e..341c54817 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,9 +1,11 @@ mod db; +mod metadata_json; mod metrics; mod plerkle_messenger_queue; mod solana_rpc; pub use db::*; +pub use metadata_json::*; pub use metrics::*; pub use plerkle_messenger_queue::*; pub use solana_rpc::*; diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs new file mode 100644 index 000000000..55c0dc64c --- /dev/null +++ b/core/src/metadata_json.rs @@ -0,0 +1,223 @@ +use { + backon::{ExponentialBuilder, Retryable}, + clap::Parser, + digital_asset_types::dao::asset_data, + futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}, + indicatif::HumanDuration, + log::{debug, error}, + reqwest::{Client, Url as ReqwestUrl}, + sea_orm::{entity::*, SqlxPostgresConnector}, + tokio::{ + sync::mpsc::{error::SendError, unbounded_channel, UnboundedSender}, + task::JoinHandle, + time::Instant, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DownloadMetadataInfo { + asset_data_id: Vec, + uri: String, +} + +impl DownloadMetadataInfo { + pub fn new(asset_data_id: Vec, uri: String) -> Self { + Self { + asset_data_id, + uri: uri.trim().replace('\0', ""), + } + } + + pub fn into_inner(self) -> (Vec, String) { + (self.asset_data_id, self.uri) + } +} + +pub type DownloadMetadataNotifier = Box< + dyn Fn( + DownloadMetadataInfo, + ) -> BoxFuture<'static, Result<(), Box>> + + Sync + + Send, +>; + +pub async fn create_download_metadata_notifier( + download_metadata_json_sender: UnboundedSender, +) -> DownloadMetadataNotifier { + Box::new(move |info: DownloadMetadataInfo| -> BoxFuture<'static, Result<(), Box>> + { + let task = download_metadata_json_sender.send(info).map_err(Into::into); + + Box::pin(async move { task }) + }) +} + +#[derive(Parser, Clone, Debug)] +pub struct MetadataJsonDownloadWorkerArgs { + /// The number of worker threads + #[arg(long, env, default_value = "25")] + metadata_json_download_worker_count: usize, + /// The request timeout in milliseconds + #[arg(long, env, default_value = "1000")] + metadata_json_download_worker_request_timeout: u64, +} + +impl MetadataJsonDownloadWorkerArgs { + pub fn start( + &self, + pool: sqlx::PgPool, + ) -> Result< + (JoinHandle<()>, UnboundedSender), + MetadataJsonDownloadWorkerError, + > { + let (sender, mut rx) = unbounded_channel::(); + let worker_count = self.metadata_json_download_worker_count; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_millis( + self.metadata_json_download_worker_request_timeout, + )) + .build()?; + + let handle = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + + while let Some(download_metadata_info) = rx.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } + + let pool = pool.clone(); + let client = client.clone(); + + handlers.push(spawn_task(client, pool, download_metadata_info)); + } + + while handlers.next().await.is_some() {} + }); + + Ok((handle, sender)) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum MetadataJsonDownloadWorkerError { + #[error("send error: {0}")] + Send(#[from] SendError), + #[error("join error: {0}")] + Join(#[from] tokio::task::JoinError), + #[error("reqwest: {0}")] + Reqwest(#[from] reqwest::Error), +} + +fn spawn_task( + client: Client, + pool: sqlx::PgPool, + download_metadata_info: DownloadMetadataInfo, +) -> JoinHandle<()> { + tokio::spawn(async move { + let timing = Instant::now(); + let asset_data_id = + bs58::encode(download_metadata_info.asset_data_id.clone()).into_string(); + + if let Err(e) = perform_metadata_json_task(client, pool, download_metadata_info).await { + error!("Asset {} failed: {}", asset_data_id, e); + } + + debug!( + "Asset {} finished in {}", + asset_data_id, + HumanDuration(timing.elapsed()) + ); + }) +} + +#[derive(thiserror::Error, Debug)] +pub enum FetchMetadataJsonError { + #[error("reqwest: {0}")] + GenericReqwest(#[from] reqwest::Error), + #[error("json parse for url({url}) with {source}")] + Parse { + source: reqwest::Error, + url: ReqwestUrl, + }, + #[error("response {status} for url ({url}) with {source}")] + Response { + source: reqwest::Error, + url: ReqwestUrl, + status: StatusCode, + }, + #[error("url parse: {0}")] + Url(#[from] url::ParseError), +} + +#[derive(Debug, derive_more::Display)] +pub enum StatusCode { + Unknown, + Code(reqwest::StatusCode), +} + +async fn fetch_metadata_json( + client: Client, + metadata_json_url: &str, +) -> Result { + (|| async { + let url = ReqwestUrl::parse(metadata_json_url)?; + + let response = client.get(url.clone()).send().await?; + + match response.error_for_status() { + Ok(res) => res + .json::() + .await + .map_err(|source| FetchMetadataJsonError::Parse { source, url }), + Err(source) => { + let status = source + .status() + .map(StatusCode::Code) + .unwrap_or(StatusCode::Unknown); + + Err(FetchMetadataJsonError::Response { + source, + url, + status, + }) + } + } + }) + .retry(&ExponentialBuilder::default()) + .await +} + +#[derive(thiserror::Error, Debug)] +pub enum MetadataJsonTaskError { + #[error("sea orm: {0}")] + SeaOrm(#[from] sea_orm::DbErr), + #[error("metadata json: {0}")] + Fetch(#[from] FetchMetadataJsonError), + #[error("asset not found in the db")] + AssetNotFound, +} + +pub async fn perform_metadata_json_task( + client: Client, + pool: sqlx::PgPool, + download_metadata_info: DownloadMetadataInfo, +) -> Result { + match fetch_metadata_json(client, &download_metadata_info.uri).await { + Ok(metadata) => { + let active_model = asset_data::ActiveModel { + id: Set(download_metadata_info.asset_data_id), + metadata: Set(metadata), + reindex: Set(Some(false)), + ..Default::default() + }; + + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + + let model = active_model.update(&conn).await?; + + Ok(model) + } + Err(e) => Err(MetadataJsonTaskError::Fetch(e)), + } +} diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index fd17a2bed..0b8494cd1 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -14,6 +14,7 @@ cadence = { workspace = true } cadence-macros = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive", "cargo"] } +das-core = { workspace = true } digital_asset_types = { workspace = true, features = [ "json_types", "sql_types", diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs index 17ec935a0..a74c650be 100644 --- a/nft_ingester/src/tasks/common/mod.rs +++ b/nft_ingester/src/tasks/common/mod.rs @@ -2,10 +2,10 @@ use { super::{BgTask, FromTaskData, IngesterError, IntoTaskData, TaskData}, async_trait::async_trait, chrono::{NaiveDateTime, Utc}, + das_core::{DownloadMetadataInfo, DownloadMetadataNotifier}, digital_asset_types::dao::asset_data, futures::future::BoxFuture, log::debug, - program_transformers::{DownloadMetadataInfo, DownloadMetadataNotifier}, reqwest::{Client, ClientBuilder}, sea_orm::*, serde::{Deserialize, Serialize}, diff --git a/ops/Cargo.toml b/ops/Cargo.toml index 48c0750e1..63b91de67 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -11,12 +11,12 @@ name = "das-ops" [dependencies] anchor-client = { workspace = true } anyhow = { workspace = true } -backon = { workspace = true } borsh = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "env"] } +das-backfill = { workspace = true } das-core = { workspace = true } digital_asset_types = { workspace = true } env_logger = { workspace = true } diff --git a/ops/src/account/account_info.rs b/ops/src/account/account_info.rs index b6216b331..6ce48a544 100644 --- a/ops/src/account/account_info.rs +++ b/ops/src/account/account_info.rs @@ -17,7 +17,7 @@ pub async fn fetch(rpc: &Rpc, pubkey: Pubkey) -> Result Result<()> { let pool = connect_db(&config.database).await?; let num_workers = config.account_worker_count; + let metadata_json_download_db_pool = pool.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = config + .metadata_json_download_worker + .start(metadata_json_download_db_pool)?; + let (tx, mut rx) = mpsc::channel::>(config.max_buffer_size); + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender.clone()).await; let mut workers = FuturesUnordered::new(); - let program_transformer = Arc::new(ProgramTransformer::new( - pool, - Box::new(|_info| ready(Ok(())).boxed()), - false, - )); + let program_transformer = Arc::new(ProgramTransformer::new(pool, download_metadata_notifier)); let account_info_worker_manager = tokio::spawn(async move { while let Some(account_infos) = rx.recv().await { @@ -102,5 +108,9 @@ pub async fn run(config: Args) -> Result<()> { account_info_worker_manager.await?; + drop(metadata_json_download_sender); + + metadata_json_download_worker.await?; + Ok(()) } diff --git a/ops/src/account/single.rs b/ops/src/account/single.rs index a0482b862..81d6e0ad4 100644 --- a/ops/src/account/single.rs +++ b/ops/src/account/single.rs @@ -2,8 +2,10 @@ use anyhow::Result; use super::account_info; use clap::Parser; -use das_core::{connect_db, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; -use futures::future::{ready, FutureExt}; +use das_core::{ + connect_db, create_download_metadata_notifier, MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, + SolanaRpcArgs, +}; use program_transformers::ProgramTransformer; use solana_sdk::pubkey::Pubkey; @@ -13,9 +15,8 @@ pub struct Args { #[clap(flatten)] pub database: PoolArgs, - /// Metrics configuration #[clap(flatten)] - pub metrics: MetricsArgs, + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, /// Solana configuration #[clap(flatten)] @@ -32,15 +33,26 @@ fn parse_pubkey(s: &str) -> Result { pub async fn run(config: Args) -> Result<()> { let rpc = Rpc::from_config(&config.solana); let pool = connect_db(&config.database).await?; + let metadata_json_download_db_pool = pool.clone(); - let program_transformer = - ProgramTransformer::new(pool, Box::new(|_info| ready(Ok(())).boxed()), false); + let (metadata_json_download_worker, metadata_json_download_sender) = config + .metadata_json_download_worker + .start(metadata_json_download_db_pool)?; - let account_info = account_info::fetch(&rpc, config.account).await?; + { + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender).await; - program_transformer - .handle_account_update(&account_info) - .await?; + let program_transformer = ProgramTransformer::new(pool, download_metadata_notifier); + + let account_info = account_info::fetch(&rpc, config.account).await?; + + program_transformer + .handle_account_update(&account_info) + .await?; + } + + metadata_json_download_worker.await?; Ok(()) } diff --git a/ops/src/bubblegum/README.md b/ops/src/bubblegum/README.md index 397dbb234..20f4bf235 100644 --- a/ops/src/bubblegum/README.md +++ b/ops/src/bubblegum/README.md @@ -10,62 +10,4 @@ Command line arguments can also be set through environment variables. The `backfill` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL. -**warning**: The command expects full archive access to transactions. Before proceeding ensure your RPC is able to serve complete transaction history for Solana. - -``` -Usage: das-ops bubblegum backfill [OPTIONS] --database-url --messenger-redis-url --solana-rpc-url - -Options: - --tree-crawler-count - Number of tree crawler workers [env: TREE_CRAWLER_COUNT=] [default: 20] - --signature-channel-size - The size of the signature channel [env: SIGNATURE_CHANNEL_SIZE=] [default: 10000] - --gap-channel-size - The size of the signature channel [env: GAP_CHANNEL_SIZE=] [default: 1000] - --transaction-worker-count - The number of transaction workers [env: TRANSACTION_WORKER_COUNT=] [default: 100] - --gap-worker-count - The number of gap workers [env: GAP_WORKER_COUNT=] [default: 25] - --only-trees - The list of trees to crawl. If not specified, all trees will be crawled [env: ONLY_TREES=] - --database-url - The database URL [env: DATABASE_URL=] - --database-max-connections - The maximum number of connections to the database [env: DATABASE_MAX_CONNECTIONS=] [default: 125] - --database-min-connections - The minimum number of connections to the database [env: DATABASE_MIN_CONNECTIONS=] [default: 5] - --messenger-redis-url - [env: MESSENGER_REDIS_URL=] - --messenger-redis-batch-size - [env: MESSENGER_REDIS_BATCH_SIZE=] [default: 100] - --messenger-queue-connections - [env: MESSENGER_QUEUE_CONNECTIONS=] [default: 25] - --messenger-queue-stream - [env: MESSENGER_QUEUE_STREAM=] [default: TXNFILL] - --metrics-host - [env: METRICS_HOST=] [default: 127.0.0.1] - --metrics-port - [env: METRICS_PORT=] [default: 8125] - --metrics-prefix - [env: METRICS_PREFIX=] [default: das.backfiller] - --solana-rpc-url - [env: SOLANA_RPC_URL=] - -h, --help - Print help -``` - -### Metrics - -The bubblegum command provides several metrics for monitoring performance and status: - -Metric | Description ---- | --- -transaction.failed | Count of failed transaction -transaction.succeeded | Count of successfully queued transaction -transaction.queued | Time for a transaction to be queued -gap.failed | Count of failed gap crawling -gap.succeeded | Count of successfully crawled gaps -gap.queued | Time for a gap to be queued -tree.succeeded | Count of completed tree crawl -tree.crawled | Time to crawl a tree -job.completed | Time to complete the job +**warning**: The command expects full archive access to transactions. Before proceeding ensure your RPC is able to serve complete transaction history for Solana. \ No newline at end of file diff --git a/ops/src/bubblegum/audit.rs b/ops/src/bubblegum/audit.rs deleted file mode 100644 index deee3c4dd..000000000 --- a/ops/src/bubblegum/audit.rs +++ /dev/null @@ -1,89 +0,0 @@ -use anyhow::Result; -use clap::Parser; -use das_core::{connect_db, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; -use digital_asset_types::dao::cl_audits_v2; -use futures::future; -use sea_orm::{CursorTrait, EntityTrait, SqlxPostgresConnector}; -use solana_sdk::signature::Signature; -use solana_transaction_status::EncodedConfirmedTransactionWithStatusMeta; - -use tokio::io::{stdout, AsyncWriteExt}; - -#[derive(Debug, Parser, Clone)] -pub struct Args { - /// Database configuration - #[clap(flatten)] - pub database: PoolArgs, - - /// Metrics configuration - #[clap(flatten)] - pub metrics: MetricsArgs, - - /// Solana configuration - #[clap(flatten)] - pub solana: SolanaRpcArgs, - - #[arg(long, env, default_value = "10000")] - pub batch_size: u64, -} - -/// The `audit` commands checks `cl_audits_v2` for any failed transactions and logs them to stdout. -pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(&config.database).await?; - - let solana_rpc = Rpc::from_config(&config.solana); - - let mut output = stdout(); - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - let mut after = None; - - loop { - let mut query = cl_audits_v2::Entity::find().cursor_by(cl_audits_v2::Column::Id); - let mut query = query.first(config.batch_size); - - if let Some(after) = after { - query = query.after(after); - } - - let entries = query.all(&conn).await?; - - let mut transactions = vec![]; - - for entry in entries.clone() { - transactions.push(fetch_transaction(entry, solana_rpc.clone())); - } - - let transactions = future::join_all(transactions).await; - - for (signature, transaction) in transactions.into_iter().flatten() { - if let Some(meta) = transaction.transaction.meta { - if meta.err.is_some() { - output - .write_all(format!("{}\n", signature).as_bytes()) - .await?; - - output.flush().await?; - } - } - } - - after = entries.last().map(|cl_audit_v2| cl_audit_v2.id); - - if entries.is_empty() { - break; - } - } - - Ok(()) -} - -async fn fetch_transaction( - entry: cl_audits_v2::Model, - solana_rpc: Rpc, -) -> Result<(Signature, EncodedConfirmedTransactionWithStatusMeta)> { - let signature = Signature::try_from(entry.tx.as_ref())?; - - let transaction = solana_rpc.get_transaction(&signature).await?; - - Ok((signature, transaction)) -} diff --git a/ops/src/bubblegum/backfiller.rs b/ops/src/bubblegum/backfiller.rs index fbfb3bf7c..a53996e64 100644 --- a/ops/src/bubblegum/backfiller.rs +++ b/ops/src/bubblegum/backfiller.rs @@ -1,64 +1,18 @@ -use super::{ - tree::{TreeGapFill, TreeGapModel, TreeResponse}, - BubblegumOpsErrorKind, -}; use anyhow::Result; -use cadence_macros::{statsd_count, statsd_time}; use clap::Parser; -use das_core::{connect_db, setup_metrics, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; -use digital_asset_types::dao::cl_audits_v2; -use futures::future::{ready, FutureExt}; -use futures::{stream::FuturesUnordered, StreamExt}; -use indicatif::HumanDuration; -use log::{debug, error}; -use program_transformers::{ProgramTransformer, TransactionInfo}; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector}; -use solana_program::pubkey::Pubkey; -use solana_sdk::instruction::CompiledInstruction; -use solana_sdk::signature::Signature; -use solana_sdk::transaction::VersionedTransaction; -use solana_transaction_status::{ - option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta, - InnerInstruction, InnerInstructions, UiInstruction, -}; -use sqlx::PgPool; -use std::time::Instant; -use tokio::{sync::mpsc, task::JoinHandle}; +use das_backfill::{start_bubblegum_backfill, BubblegumBackfillArgs, BubblegumBackfillContext}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Number of tree crawler workers - #[arg(long, env, default_value = "20")] - pub tree_crawler_count: usize, - - /// The size of the signature channel. - #[arg(long, env, default_value = "100000")] - pub signature_channel_size: usize, - - /// The size of the signature channel. - #[arg(long, env, default_value = "1000")] - pub gap_channel_size: usize, - - /// The number of transaction workers. - #[arg(long, env, default_value = "50")] - pub transaction_worker_count_per_tree: usize, - - /// The number of gap workers. - #[arg(long, env, default_value = "25")] - pub gap_worker_count: usize, - - /// The list of trees to crawl. If not specified, all trees will be crawled. - #[arg(long, env, use_value_delimiter = true)] - pub only_trees: Option>, + /// Backfill Bubblegum Args + #[clap(flatten)] + pub backfill_bubblegum: BubblegumBackfillArgs, /// Database configuration #[clap(flatten)] pub database: PoolArgs, - /// Metrics configuration - #[clap(flatten)] - pub metrics: MetricsArgs, - /// Solana configuration #[clap(flatten)] pub solana: SolanaRpcArgs, @@ -66,17 +20,14 @@ pub struct Args { /// Executes the backfilling operation for the tree crawler. /// -/// This function sets up the essential components required for the backfilling operation, -/// including database connections, RPC clients, and worker managers to handle -/// transactions and gaps. It retrieves the necessary trees for crawling and orchestrates -/// the crawling operation across various workers. +/// This function initializes the necessary components for the backfilling operation, +/// such as database connections and RPC clients, and then delegates the actual +/// backfilling logic to the `das_bubblegum_backfill` crate. /// /// The function undertakes the following key tasks: /// - Establishes database connections and initializes RPC clients. -/// - Configures channels for inter-component communication. -/// - Deploys worker managers to handle transactions and gaps. -/// - Retrieves trees from the database and oversees their crawling. -/// - Monitors metrics and logs activities throughout the operation. +/// - Creates a context for the backfilling operation. +/// - Invokes the `start_bubblegum_backfill` function from the `das_bubblegum_backfill` crate. /// /// # Arguments /// @@ -91,378 +42,12 @@ pub struct Args { /// /// # Errors /// -/// Potential errors can arise from database connectivity issues, RPC failures, -/// or complications in spawning and managing worker tasks. +/// Potential errors can arise from database connectivity issues or RPC failures. pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(&config.database).await?; + let database_pool = connect_db(&config.database).await?; let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumBackfillContext::new(database_pool, solana_rpc); - setup_metrics(&config.metrics)?; - - let started = Instant::now(); - - let trees = if let Some(ref only_trees) = config.only_trees { - TreeResponse::find(&solana_rpc, only_trees.clone()).await? - } else { - TreeResponse::all(&solana_rpc).await? - }; - - let tree_count = trees.len(); - - debug!( - "fetched {} trees in {}", - tree_count, - HumanDuration(started.elapsed()) - ); - - let tree_crawler_count = config.tree_crawler_count; - let mut crawl_handles = FuturesUnordered::new(); - - for tree in trees { - if crawl_handles.len() >= tree_crawler_count { - crawl_handles.next().await; - } - - let pool = pool.clone(); - let solana_rpc = solana_rpc.clone(); - - let handle = spawn_tree_worker(&config, pool, solana_rpc, tree); - - crawl_handles.push(handle); - } - - futures::future::try_join_all(crawl_handles).await?; - - statsd_time!("job.completed", started.elapsed()); - - debug!( - "crawled {} trees in {}", - tree_count, - HumanDuration(started.elapsed()) - ); - - Ok(()) -} - -fn spawn_tree_worker( - config: &Args, - pool: PgPool, - rpc: Rpc, - tree: TreeResponse, -) -> JoinHandle> { - let config = config.clone(); - let gap_solana_rpc = rpc.clone(); - let gap_pool = pool.clone(); - - tokio::spawn(async move { - let timing = Instant::now(); - - let transaction_worker_count = config.transaction_worker_count_per_tree; - - let (sig_sender, mut sig_receiver) = - mpsc::channel::(config.signature_channel_size); - let gap_sig_sender = sig_sender.clone(); - - let (gap_sender, mut gap_receiver) = mpsc::channel::(config.gap_channel_size); - let (transaction_sender, mut transaction_receiver) = - mpsc::channel::(config.signature_channel_size); - - let signature_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - - while let Some(signature) = sig_receiver.recv().await { - if handlers.len() >= transaction_worker_count { - handlers.next().await; - } - - let solana_rpc = rpc.clone(); - let transaction_sender = transaction_sender.clone(); - - let handle = spawn_transaction_worker(solana_rpc, transaction_sender, signature); - - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - - drop(transaction_sender); - }); - - let gap_worker_count = config.gap_worker_count; - - let gap_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - let sender = gap_sig_sender.clone(); - - while let Some(gap) = gap_receiver.recv().await { - if handlers.len() >= gap_worker_count { - handlers.next().await; - } - - let client = gap_solana_rpc.clone(); - let sender = sender.clone(); - - let handle = spawn_crawl_worker(client, sender, gap); - - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - - drop(sig_sender); - }); - - let transaction_worker_manager = tokio::spawn(async move { - let mut transactions = Vec::new(); - let pool = pool.clone(); - - let program_transformer = - ProgramTransformer::new(pool, Box::new(|_info| ready(Ok(())).boxed()), true); - - while let Some(gap) = transaction_receiver.recv().await { - transactions.push(gap); - } - - transactions.sort_by(|a, b| b.signature.cmp(&a.signature)); - - for transaction in transactions { - if let Err(e) = program_transformer.handle_transaction(&transaction).await { - error!("handle transaction: {:?}", e) - }; - } - }); - - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(gap_pool); - - let mut gaps = TreeGapModel::find(&conn, tree.pubkey) - .await? - .into_iter() - .map(TryInto::try_into) - .collect::, _>>()?; - - let upper_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_desc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; - - let lower_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_asc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; - - drop(conn); - - if let Some(upper_seq) = upper_known_seq { - let signature = Signature::try_from(upper_seq.tx.as_ref())?; - - gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); - } else if tree.seq > 0 { - gaps.push(TreeGapFill::new(tree.pubkey, None, None)); - } - - if let Some(lower_seq) = lower_known_seq.filter(|seq| seq.seq > 1) { - let signature = Signature::try_from(lower_seq.tx.as_ref())?; - - gaps.push(TreeGapFill::new(tree.pubkey, Some(signature), None)); - } - - for gap in gaps { - if let Err(e) = gap_sender.send(gap).await { - statsd_count!("gap.failed", 1); - error!("send gap: {:?}", e); - } - } - - drop(gap_sender); - gap_worker_manager.await?; - - signature_worker_manager.await?; - - transaction_worker_manager.await?; - - statsd_count!("tree.succeeded", 1); - statsd_time!("tree.crawled", timing.elapsed()); - - Ok::<(), anyhow::Error>(()) - }) -} - -fn spawn_crawl_worker( - client: Rpc, - sender: mpsc::Sender, - gap: TreeGapFill, -) -> JoinHandle<()> { - tokio::spawn(async move { - let timing = Instant::now(); - - if let Err(e) = gap.crawl(client, sender).await { - error!("tree transaction: {:?}", e); - - statsd_count!("gap.failed", 1); - } else { - statsd_count!("gap.succeeded", 1); - } - - statsd_time!("gap.queued", timing.elapsed()); - }) -} - -pub struct FetchedEncodedTransactionWithStatusMeta(pub EncodedConfirmedTransactionWithStatusMeta); - -impl TryFrom for TransactionInfo { - type Error = BubblegumOpsErrorKind; - - fn try_from( - fetched_transaction: FetchedEncodedTransactionWithStatusMeta, - ) -> Result { - let mut account_keys = Vec::new(); - let encoded_transaction_with_status_meta = fetched_transaction.0; - - let ui_transaction: VersionedTransaction = encoded_transaction_with_status_meta - .transaction - .transaction - .decode() - .ok_or(BubblegumOpsErrorKind::Generic( - "unable to decode transaction".to_string(), - ))?; - - let signature = ui_transaction.signatures[0]; - - let msg = ui_transaction.message; - - let meta = encoded_transaction_with_status_meta - .transaction - .meta - .ok_or(BubblegumOpsErrorKind::Generic( - "unable to get meta from transaction".to_string(), - ))?; - - for address in msg.static_account_keys().iter().copied() { - account_keys.push(address); - } - let ui_loaded_addresses = meta.loaded_addresses; - - let message_address_table_lookup = msg.address_table_lookups(); - - if message_address_table_lookup.is_some() { - if let OptionSerializer::Some(ui_lookup_table) = ui_loaded_addresses { - for address in ui_lookup_table.writable { - account_keys.push(PubkeyString(address).try_into()?); - } - - for address in ui_lookup_table.readonly { - account_keys.push(PubkeyString(address).try_into()?); - } - } - } - - let mut meta_inner_instructions = Vec::new(); - - let compiled_instruction = msg.instructions().to_vec(); - - let mut instructions = Vec::new(); - - for inner in compiled_instruction { - instructions.push(InnerInstruction { - stack_height: Some(0), - instruction: CompiledInstruction { - program_id_index: inner.program_id_index, - accounts: inner.accounts, - data: inner.data, - }, - }); - } - - meta_inner_instructions.push(InnerInstructions { - index: 0, - instructions, - }); - - if let OptionSerializer::Some(inner_instructions) = meta.inner_instructions { - for ix in inner_instructions { - let mut instructions = Vec::new(); - - for inner in ix.instructions { - if let UiInstruction::Compiled(compiled) = inner { - instructions.push(InnerInstruction { - stack_height: compiled.stack_height, - instruction: CompiledInstruction { - program_id_index: compiled.program_id_index, - accounts: compiled.accounts, - data: bs58::decode(compiled.data) - .into_vec() - .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?, - }, - }); - } - } - - meta_inner_instructions.push(InnerInstructions { - index: ix.index, - instructions, - }); - } - } - - Ok(Self { - slot: encoded_transaction_with_status_meta.slot, - account_keys, - signature, - message_instructions: msg.instructions().to_vec(), - meta_inner_instructions, - }) - } -} - -async fn queue_transaction<'a>( - client: Rpc, - sender: mpsc::Sender, - signature: Signature, -) -> Result<(), BubblegumOpsErrorKind> { - let transaction = client.get_transaction(&signature).await?; - - sender - .send(FetchedEncodedTransactionWithStatusMeta(transaction).try_into()?) - .await - .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?; - - Ok(()) -} - -fn spawn_transaction_worker( - client: Rpc, - sender: mpsc::Sender, - signature: Signature, -) -> JoinHandle<()> { - tokio::spawn(async move { - let timing = Instant::now(); - - if let Err(e) = queue_transaction(client, sender, signature).await { - error!("queue transaction: {:?}", e); - - statsd_count!("transaction.failed", 1); - } else { - statsd_count!("transaction.succeeded", 1); - } - - statsd_time!("transaction.queued", timing.elapsed()); - }) -} - -pub struct PubkeyString(pub String); - -impl TryFrom for Pubkey { - type Error = BubblegumOpsErrorKind; - - fn try_from(value: PubkeyString) -> Result { - let decoded_bytes = bs58::decode(value.0) - .into_vec() - .map_err(|e| BubblegumOpsErrorKind::Generic(e.to_string()))?; - - Pubkey::try_from(decoded_bytes) - .map_err(|_| BubblegumOpsErrorKind::Generic("unable to convert pubkey".to_string())) - } + start_bubblegum_backfill(context, config.backfill_bubblegum).await } diff --git a/ops/src/bubblegum/cmd.rs b/ops/src/bubblegum/cmd.rs index b19715483..1004a227d 100644 --- a/ops/src/bubblegum/cmd.rs +++ b/ops/src/bubblegum/cmd.rs @@ -1,27 +1,6 @@ -use super::{audit, backfiller}; +use super::backfiller; use anyhow::Result; use clap::{Args, Subcommand}; -use thiserror::Error as ThisError; - -#[derive(ThisError, Debug)] -pub enum BubblegumOpsErrorKind { - #[error("anchor")] - Anchor(#[from] anchor_client::anchor_lang::error::Error), - #[error("solana rpc")] - Rpc(#[from] solana_client::client_error::ClientError), - #[error("parse pubkey")] - ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), - #[error("serialize tree response")] - SerializeTreeResponse, - #[error("sea orm")] - Database(#[from] sea_orm::DbErr), - #[error("try from pubkey")] - TryFromPubkey, - #[error("try from signature")] - TryFromSignature, - #[error("generic error: {0}")] - Generic(String), -} #[derive(Debug, Clone, Subcommand)] pub enum Commands { @@ -29,9 +8,6 @@ pub enum Commands { /// It crawls through trees and backfills any missed tree transactions. #[clap(name = "backfill")] Backfill(backfiller::Args), - /// The `audit` commands checks `cl_audits_v2` for any failed transactions and logs them to stdout. - #[clap(name = "audit")] - Audit(audit::Args), } #[derive(Debug, Clone, Args)] @@ -45,9 +21,6 @@ pub async fn subcommand(subcommand: BubblegumCommand) -> Result<()> { Commands::Backfill(args) => { backfiller::run(args).await?; } - Commands::Audit(args) => { - audit::run(args).await?; - } } Ok(()) diff --git a/ops/src/bubblegum/mod.rs b/ops/src/bubblegum/mod.rs index eb4c867ad..d798b6e9e 100644 --- a/ops/src/bubblegum/mod.rs +++ b/ops/src/bubblegum/mod.rs @@ -1,6 +1,4 @@ -mod audit; mod backfiller; mod cmd; -mod tree; pub use cmd::*; diff --git a/ops/src/bubblegum/tree.rs b/ops/src/bubblegum/tree.rs deleted file mode 100644 index 14685ba40..000000000 --- a/ops/src/bubblegum/tree.rs +++ /dev/null @@ -1,260 +0,0 @@ -use super::BubblegumOpsErrorKind; -use anyhow::Result; -use borsh::BorshDeserialize; -use clap::Args; -use das_core::Rpc; -use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; -use solana_client::rpc_filter::{Memcmp, RpcFilterType}; -use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; -use solana_sdk::{account::Account, pubkey::Pubkey, signature::Signature}; -use spl_account_compression::id; -use spl_account_compression::state::{ - merkle_tree_get_size, ConcurrentMerkleTreeHeader, CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1, -}; -use std::str::FromStr; -use tokio::sync::mpsc::Sender; - -const GET_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1000; - -#[derive(Debug, Clone, Args)] -pub struct ConfigBackfiller { - /// Solana RPC URL - #[arg(long, env)] - pub solana_rpc_url: String, -} - -const TREE_GAP_SQL: &str = r#" -WITH sequenced_data AS ( - SELECT - tree, - seq, - LEAD(seq) OVER (ORDER BY seq ASC) AS next_seq, - tx AS current_tx, - LEAD(tx) OVER (ORDER BY seq ASC) AS next_tx - FROM - cl_audits_v2 - WHERE - tree = $1 -), -gaps AS ( - SELECT - tree, - seq AS gap_start_seq, - next_seq AS gap_end_seq, - current_tx AS lower_bound_tx, - next_tx AS upper_bound_tx - FROM - sequenced_data - WHERE - next_seq IS NOT NULL AND - next_seq - seq > 1 -) -SELECT - tree, - gap_start_seq, - gap_end_seq, - lower_bound_tx, - upper_bound_tx -FROM - gaps -ORDER BY - gap_start_seq; -"#; - -#[derive(Debug, FromQueryResult, PartialEq, Clone)] -pub struct TreeGapModel { - pub tree: Vec, - pub gap_start_seq: i64, - pub gap_end_seq: i64, - pub lower_bound_tx: Vec, - pub upper_bound_tx: Vec, -} - -impl TreeGapModel { - pub async fn find( - conn: &DatabaseConnection, - tree: Pubkey, - ) -> Result, BubblegumOpsErrorKind> { - let statement = Statement::from_sql_and_values( - DbBackend::Postgres, - TREE_GAP_SQL, - vec![Value::Bytes(Some(Box::new(tree.as_ref().to_vec())))], - ); - - TreeGapModel::find_by_statement(statement) - .all(conn) - .await - .map_err(Into::into) - } -} - -impl TryFrom for TreeGapFill { - type Error = BubblegumOpsErrorKind; - - fn try_from(model: TreeGapModel) -> Result { - let tree = - Pubkey::try_from(model.tree).map_err(|_| BubblegumOpsErrorKind::TryFromPubkey)?; - let upper = Signature::try_from(model.upper_bound_tx) - .map_err(|_| BubblegumOpsErrorKind::TryFromSignature)?; - let lower = Signature::try_from(model.lower_bound_tx) - .map_err(|_| BubblegumOpsErrorKind::TryFromSignature)?; - - Ok(Self::new(tree, Some(upper), Some(lower))) - } -} - -pub struct TreeGapFill { - tree: Pubkey, - before: Option, - until: Option, -} - -impl TreeGapFill { - pub fn new(tree: Pubkey, before: Option, until: Option) -> Self { - Self { - tree, - before, - until, - } - } - - pub async fn crawl(&self, client: Rpc, sender: Sender) -> Result<()> { - let mut before = self.before; - - loop { - let sigs = client - .get_signatures_for_address(&self.tree, before, self.until) - .await?; - let sig_count = sigs.len(); - - let successful_transactions = sigs - .into_iter() - .filter(|transaction| transaction.err.is_none()) - .collect::>(); - - for sig in successful_transactions.iter() { - let sig = Signature::from_str(&sig.signature)?; - - sender.send(sig).await?; - - before = Some(sig); - } - - if sig_count < GET_SIGNATURES_FOR_ADDRESS_LIMIT { - break; - } - } - - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub struct TreeHeaderResponse { - pub max_depth: u32, - pub max_buffer_size: u32, - pub creation_slot: u64, - pub size: usize, -} - -impl TryFrom for TreeHeaderResponse { - type Error = BubblegumOpsErrorKind; - - fn try_from(payload: ConcurrentMerkleTreeHeader) -> Result { - let size = merkle_tree_get_size(&payload)?; - - Ok(Self { - max_depth: payload.get_max_depth(), - max_buffer_size: payload.get_max_buffer_size(), - creation_slot: payload.get_creation_slot(), - size, - }) - } -} - -#[derive(Debug, Clone)] -pub struct TreeResponse { - pub pubkey: Pubkey, - pub tree_header: TreeHeaderResponse, - pub seq: u64, -} - -impl TreeResponse { - pub fn try_from_rpc(pubkey: Pubkey, account: Account) -> Result { - let bytes = account.data.as_slice(); - - let (header_bytes, rest) = bytes.split_at(CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1); - let header: ConcurrentMerkleTreeHeader = - ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?; - - let merkle_tree_size = merkle_tree_get_size(&header)?; - let (tree_bytes, _canopy_bytes) = rest.split_at(merkle_tree_size); - - let seq_bytes = tree_bytes[0..8].try_into()?; - let seq = u64::from_le_bytes(seq_bytes); - - let (auth, _) = Pubkey::find_program_address(&[pubkey.as_ref()], &mpl_bubblegum::ID); - - header.assert_valid_authority(&auth)?; - - let tree_header = header.try_into()?; - - Ok(Self { - pubkey, - tree_header, - seq, - }) - } - - pub async fn all(client: &Rpc) -> Result, BubblegumOpsErrorKind> { - Ok(client - .get_program_accounts( - &id(), - Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes( - 0, - vec![1u8], - ))]), - ) - .await? - .into_iter() - .filter_map(|(pubkey, account)| Self::try_from_rpc(pubkey, account).ok()) - .collect()) - } - - pub async fn find( - client: &Rpc, - pubkeys: Vec, - ) -> Result, BubblegumOpsErrorKind> { - let pubkeys: Vec = pubkeys - .into_iter() - .map(|p| Pubkey::from_str(&p)) - .collect::, _>>()?; - let pubkey_batches = pubkeys.chunks(100); - let pubkey_batches_count = pubkey_batches.len(); - - let mut gma_handles = Vec::with_capacity(pubkey_batches_count); - - for batch in pubkey_batches { - gma_handles.push(async move { - let accounts = client.get_multiple_accounts(batch).await?; - - let results: Vec<(&Pubkey, Option)> = batch.iter().zip(accounts).collect(); - - Ok::<_, BubblegumOpsErrorKind>(results) - }) - } - - let result = futures::future::try_join_all(gma_handles).await?; - - let trees = result - .into_iter() - .flatten() - .filter_map(|(pubkey, account)| { - account.map(|account| Self::try_from_rpc(*pubkey, account)) - }) - .collect::, _>>() - .map_err(|_| BubblegumOpsErrorKind::SerializeTreeResponse)?; - - Ok(trees) - } -} diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index 35bab7a19..54ea735a9 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -10,7 +10,11 @@ blockbuster = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +das-core = { workspace = true } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } futures = { workspace = true } heck = { workspace = true } mpl-bubblegum = { workspace = true } diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index b317c47f6..aada07fc0 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -15,7 +15,7 @@ use { ProgramParseResult, }, }, - futures::future::BoxFuture, + das_core::{DownloadMetadataInfo, DownloadMetadataNotifier}, sea_orm::{ entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, @@ -52,33 +52,6 @@ pub struct TransactionInfo { pub meta_inner_instructions: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DownloadMetadataInfo { - asset_data_id: Vec, - uri: String, -} - -impl DownloadMetadataInfo { - pub fn new(asset_data_id: Vec, uri: String) -> Self { - Self { - asset_data_id, - uri: uri.trim().replace('\0', ""), - } - } - - pub fn into_inner(self) -> (Vec, String) { - (self.asset_data_id, self.uri) - } -} - -pub type DownloadMetadataNotifier = Box< - dyn Fn( - DownloadMetadataInfo, - ) -> BoxFuture<'static, Result<(), Box>> - + Sync - + Send, ->; - pub struct ProgramTransformer { storage: DatabaseConnection, download_metadata_notifier: DownloadMetadataNotifier, From 81f5d71df955de005be32d269c82f766785ce705 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 9 Aug 2024 11:52:30 +0200 Subject: [PATCH 28/72] chore: switch to run_v2 for ingest which handles ingesting accounts and transactions --- digital_asset_types/src/dapi/change_logs.rs | 4 +- grpc-ingest/config-ingester.yml | 3 - grpc-ingest/src/grpc.rs | 322 +++++++++++++++----- grpc-ingest/src/ingester.rs | 122 +++++++- grpc-ingest/src/main.rs | 4 +- grpc-ingest/src/prom.rs | 4 +- grpc-ingest/src/redis.rs | 4 + 7 files changed, 383 insertions(+), 80 deletions(-) diff --git a/digital_asset_types/src/dapi/change_logs.rs b/digital_asset_types/src/dapi/change_logs.rs index 7023ad12f..fdd19ddf9 100644 --- a/digital_asset_types/src/dapi/change_logs.rs +++ b/digital_asset_types/src/dapi/change_logs.rs @@ -200,7 +200,7 @@ fn build_asset_proof( tree_id: Vec, leaf_node_idx: i64, leaf_hash: Vec, - req_indexes: &Vec, + req_indexes: &[i64], required_nodes: &[SimpleChangeLog], ) -> AssetProof { let mut final_node_list = vec![SimpleChangeLog::default(); req_indexes.len()]; @@ -211,7 +211,7 @@ fn build_asset_proof( } for (i, (n, nin)) in final_node_list .iter_mut() - .zip(req_indexes.clone()) + .zip(req_indexes.to_owned()) .enumerate() { if *n == SimpleChangeLog::default() { diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 0dde2dfd4..123d2e16f 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -13,9 +13,6 @@ redis: xack_batch_max_size: 100 xack_batch_max_idle_ms: 10 xack_max_in_process: 100 - - type: metadata_json - stream: METADATA_JSONS - data_key: data prefetch_queue_size: 1_000 # max number of messages available in the read queue for processing xpending_max: 250 # used for reading pending messages xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index af031318f..97ba2b406 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -14,6 +14,7 @@ use { Value as RedisValue, }, serde::de, + sqlx::pool, std::{collections::HashMap, num::NonZeroUsize, sync::Arc, time::Duration}, tokio::{ spawn, @@ -37,11 +38,11 @@ use { }; enum GrpcJob { - FlushRedisPipe(Arc>, MultiplexedConnection), - ProcessSubscribeUpdate(Arc>, SubscribeUpdate), + FlushRedisPipe, + ProcessSubscribeUpdate(Box), } -pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { +pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { let redis_client = redis::Client::open(config.redis.url.clone())?; let config = Arc::new(config); let connection = redis_client.get_multiplexed_tokio_connection().await?; @@ -74,99 +75,103 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { .await .context("failed to connect to gRPC")?; - let (_subscribe_tx, mut stream) = dragon_mouth_client + let (_subscribe_tx, stream) = dragon_mouth_client .subscribe_with_request(Some(request)) .await?; + tokio::pin!(stream); let pool_config = Arc::clone(&config); + let pool_connection = connection.clone(); + let pool_pipe = Arc::clone(&pipe); - let exec = Executor::builder(Nonblock(Tokio)) - .num_threads(None) - .build(move |update, _| { - let config = pool_config.clone(); + let exec = + Executor::builder(Nonblock(Tokio)) + .num_threads(None) + .build(move |update, _handle| { + let config = Arc::clone(&pool_config); + let connection = pool_connection.clone(); + let pipe = Arc::clone(&pool_pipe); - async move { - match update { - GrpcJob::FlushRedisPipe(pipe, connection) => { - let mut pipe = pipe.lock().await; - let mut connection = connection; + async move { + match update { + GrpcJob::FlushRedisPipe => { + let mut pipe = pipe.lock().await; + let mut connection = connection; - let flush = pipe.flush(&mut connection).await; + let flush = pipe.flush(&mut connection).await; - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let counts = flush.as_ref().unwrap_or_else(|counts| counts); + + for (stream, count) in counts.iter() { + redis_xadd_status_inc(stream, status, *count); + } - for (stream, count) in counts.iter() { - redis_xadd_status_inc(stream, status, count); + debug!(message = "Redis pipe flushed", ?status, ?counts); } + GrpcJob::ProcessSubscribeUpdate(update) => { + let accounts_stream = config.accounts.stream.clone(); + let accounts_stream_maxlen = config.accounts.stream_maxlen; + let accounts_stream_data_key = config.accounts.stream_data_key.clone(); + let transactions_stream = config.transactions.stream.clone(); + let transactions_stream_maxlen = config.transactions.stream_maxlen; + let transactions_stream_data_key = + config.transactions.stream_data_key.clone(); - debug!(message = "Redis pipe flushed", ?status, ?counts); - } - GrpcJob::ProcessSubscribeUpdate(pipe, update) => { - let accounts_stream = config.accounts.stream.clone(); - let accounts_stream_maxlen = config.accounts.stream_maxlen; - let accounts_stream_data_key = config.accounts.stream_data_key.clone(); - let transactions_stream = config.transactions.stream.clone(); - let transactions_stream_maxlen = config.transactions.stream_maxlen; - let transactions_stream_data_key = - config.transactions.stream_data_key.clone(); - - let SubscribeUpdate { update_oneof, .. } = update; - let mut pipe = pipe.lock().await; - - if let Some(update) = update_oneof { - match update { - UpdateOneof::Account(account) => { - pipe.xadd_maxlen( - &accounts_stream, - StreamMaxlen::Approx(accounts_stream_maxlen), - "*", - &[(&accounts_stream_data_key, account.encode_to_vec())], - ); - - debug!( - message = "Account update", - ?account, - ?accounts_stream, - ?accounts_stream_maxlen - ); - } - UpdateOneof::Transaction(transaction) => { - pipe.xadd_maxlen( - &transactions_stream, - StreamMaxlen::Approx(transactions_stream_maxlen), - "*", - &[( - &transactions_stream_data_key, - transaction.encode_to_vec(), - )], - ); - - debug!(message = "Transaction update", ?transaction,); + let SubscribeUpdate { update_oneof, .. } = *update; + + let mut pipe = pipe.lock().await; + + if let Some(update) = update_oneof { + match update { + UpdateOneof::Account(account) => { + pipe.xadd_maxlen( + &accounts_stream, + StreamMaxlen::Approx(accounts_stream_maxlen), + "*", + &[(&accounts_stream_data_key, account.encode_to_vec())], + ); + + debug!(message = "Account update", ?account,); + } + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &transactions_stream, + StreamMaxlen::Approx(transactions_stream_maxlen), + "*", + &[( + &transactions_stream_data_key, + transaction.encode_to_vec(), + )], + ); + + debug!(message = "Transaction update", ?transaction); + } + var => warn!(message = "Unknown update variant", ?var), } - var => warn!(message = "Unknown update variant", ?var), + } + + if pipe.size() >= config.redis.pipeline_max_size { + // handle.push(GrpcJob::FlushRedisPipe); } } } } - } - })?; + })?; - let deadline_pipe = Arc::clone(&pipe); let deadline_config = Arc::clone(&config); - let deadline_connection = connection.clone(); loop { tokio::select! { _ = sleep(deadline_config.redis.pipeline_max_idle) => { - exec.push(GrpcJob::FlushRedisPipe(deadline_pipe.clone(), deadline_connection.clone())); + exec.push(GrpcJob::FlushRedisPipe); } Some(Ok(msg)) = stream.next() => { debug!(message = "Received gRPC message", ?msg); - exec.push(GrpcJob::ProcessSubscribeUpdate(Arc::clone(&pipe), msg)); + exec.push(GrpcJob::ProcessSubscribeUpdate(Box::new(msg))); } _ = shutdown.next() => { - exec.push(GrpcJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); + exec.push(GrpcJob::FlushRedisPipe); break; } } @@ -176,3 +181,182 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { Ok(()) } + +pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { + let config = Arc::new(config); + let (tx, mut rx) = mpsc::channel::(config.geyser_update_message_buffer_size); // Adjust buffer size as needed + + // Connect to Redis + let client = redis::Client::open(config.redis.url.clone())?; + let connection = client.get_multiplexed_tokio_connection().await?; + + // Check stream length for the metrics + let jh_metrics_xlen = spawn({ + let connection = connection.clone(); + let streams = vec![ + config.accounts.stream.clone(), + config.transactions.stream.clone(), + ]; + async move { metrics_xlen(connection, &streams).await } + }); + tokio::pin!(jh_metrics_xlen); + + // Spawn gRPC client connections + let config = Arc::clone(&config); + let mut tx = tx.clone(); + + let mut client = GeyserGrpcClient::build_from_shared(config.geyser_endpoint.clone())? + .x_token(config.x_token.clone())? + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(10)) + .connect() + .await + .context("failed to connect to gRPC")?; + + let grc_config = Arc::clone(&config); + spawn(async move { + let mut accounts = HashMap::with_capacity(1); + let mut transactions = HashMap::with_capacity(1); + + accounts.insert( + "das".to_string(), + grc_config.accounts.filter.clone().to_proto(), + ); + transactions.insert( + "das".to_string(), + grc_config.transactions.filter.clone().to_proto(), + ); + + let request = SubscribeRequest { + accounts, + transactions, + ..Default::default() + }; + + let (_subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; + + while let Some(Ok(msg)) = stream.next().await { + if let Some(update) = msg.update_oneof { + tx.send(update) + .await + .expect("Failed to send update to management thread"); + } + } + Ok::<(), anyhow::Error>(()) + }); + + // Management thread + let mut shutdown = create_shutdown()?; + let mut tasks = JoinSet::new(); + let mut pipe = redis::pipe(); + let mut pipe_accounts = 0; + let mut pipe_transactions = 0; + + let pipeline_max_idle = config.redis.pipeline_max_idle; + let deadline = sleep(pipeline_max_idle); + tokio::pin!(deadline); + + let result = loop { + tokio::select! { + result = &mut jh_metrics_xlen => match result { + Ok(Ok(_)) => unreachable!(), + Ok(Err(error)) => break Err(error), + Err(error) => break Err(error.into()), + }, + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); + break Ok(()); + }, + Some(update) = rx.next() => { + match update { + UpdateOneof::Account(account) => { + + pipe.xadd_maxlen( + &config.accounts.stream, + StreamMaxlen::Approx(config.accounts.stream_maxlen), + "*", + &[(&config.accounts.stream_data_key, account.encode_to_vec())], + ); + + pipe_accounts += 1; + } + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &config.transactions.stream, + StreamMaxlen::Approx(config.transactions.stream_maxlen), + "*", + &[(&config.transactions.stream_data_key, transaction.encode_to_vec())] + ); + + pipe_transactions += 1; + } + _ => continue, + } + if pipe_accounts + pipe_transactions >= config.redis.pipeline_max_size { + let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); + let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); + let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); + deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); + + tasks.spawn({ + let mut connection = connection.clone(); + let config = Arc::clone(&config); + async move { + let result: RedisResult = + pipe.atomic().query_async(&mut connection).await; + + let status = result.map(|_| ()).map_err(|_| ()); + redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); + redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); + + Ok::<(), anyhow::Error>(()) + } + }); + } + }, + _ = &mut deadline => { + if pipe_accounts + pipe_transactions > 0 { + let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); + let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); + let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); + deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); + + tasks.spawn({ + let mut connection = connection.clone(); + let config = Arc::clone(&config); + async move { + let result: RedisResult = + pipe.atomic().query_async(&mut connection).await; + + let status = result.map(|_| ()).map_err(|_| ()); + redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); + redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); + + Ok::<(), anyhow::Error>(()) + } + }); + } + }, + }; + + while tasks.len() >= config.redis.max_xadd_in_process { + if let Some(result) = tasks.join_next().await { + result??; + } + } + }; + + tokio::select! { + Some(signal) = shutdown.next() => { + anyhow::bail!("{signal} received, force shutdown..."); + } + result = async move { + while let Some(result) = tasks.join_next().await { + result??; + } + Ok::<(), anyhow::Error>(()) + } => result?, + }; + + result +} diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index d0fd4ab69..d79364ee3 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -7,7 +7,7 @@ use { download_metadata_inserted_total_inc, program_transformer_task_status_inc, program_transformer_tasks_total_set, ProgramTransformerTaskStatusKind, }, - redis::{metrics_xlen, ProgramTransformerInfo, RedisStream}, + redis::{metrics_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo}, util::create_shutdown, }, chrono::Utc, @@ -18,6 +18,7 @@ use { future::{pending, BoxFuture, FusedFuture, FutureExt}, stream::StreamExt, }, + opentelemetry_sdk::trace::Config, program_transformers::{error::ProgramTransformerError, ProgramTransformer}, sea_orm::{ entity::{ActiveModelTrait, ActiveValue}, @@ -36,9 +37,125 @@ use { task::JoinSet, time::{sleep, Duration}, }, - tracing::warn, + topograph::{ + executor::{Executor, Nonblock, Tokio}, + prelude::*, + }, + tracing::{error, warn}, }; +enum IngestJob { + SaveMessage(RedisStreamMessageInfo), +} + +pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { + let client = redis::Client::open(config.redis.url.clone())?; + let connection = client.get_multiplexed_tokio_connection().await?; + let pool = pg_create_pool(config.postgres).await?; + + let (mut redis_messages, redis_tasks_fut) = RedisStream::new(config.redis, connection).await?; + tokio::pin!(redis_tasks_fut); + + let pt_accounts = Arc::new(ProgramTransformer::new( + pool.clone(), + create_download_metadata_notifier(pool.clone(), config.download_metadata)?, + )); + let pt_transactions = Arc::new(ProgramTransformer::new( + pool.clone(), + create_download_metadata_notifier(pool.clone(), config.download_metadata)?, + )); + + let exec = + Executor::builder(Nonblock(Tokio)) + .num_threads(None) + .build(move |update, _handle| { + let pt_accounts = Arc::clone(&pt_accounts); + let pt_transactions = Arc::clone(&pt_transactions); + + async move { + match update { + IngestJob::SaveMessage(msg) => { + let result = match &msg.get_data() { + ProgramTransformerInfo::Account(account) => { + pt_accounts.handle_account_update(account).await + } + ProgramTransformerInfo::Transaction(transaction) => { + pt_transactions.handle_transaction(transaction).await + } + }; + match result { + Ok(()) => program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::Success, + ), + Err(ProgramTransformerError::NotImplemented) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::NotImplemented, + ); + error!("not implemented") + } + Err(ProgramTransformerError::DeserializationError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::DeserializationError, + ); + + error!("failed to deserialize {:?}", error) + } + Err(ProgramTransformerError::ParsingError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::ParsingError, + ); + + error!("failed to parse {:?}", error) + } + Err(ProgramTransformerError::DatabaseError(error)) => { + error!("database error for {:?}", error) + } + Err(ProgramTransformerError::AssetIndexError(error)) => { + error!("indexing error for {:?}", error) + } + Err(error) => { + error!("failed to handle {:?}", error) + } + } + + let _ = msg.ack(); + + () + } + } + } + })?; + + let mut shutdown = create_shutdown()?; + + loop { + tokio::select! { + Some(msg) = redis_messages.recv() => { + exec.push(IngestJob::SaveMessage(msg)); + } + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); + break; + } + result = &mut redis_tasks_fut => { + if let Err(error) = result { + error!("Error in redis_tasks_fut: {:?}", error); + } + break; + } + } + } + + redis_messages.shutdown(); + + exec.join_async().await; + + pool.close().await; + + Ok::<(), anyhow::Error>(()) +} + +#[allow(dead_code)] pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // connect to Redis let client = redis::Client::open(config.redis.url.clone())?; @@ -83,6 +200,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { tokio::spawn({ let pt_tasks_len = Arc::clone(&pt_tasks_len); + async move { loop { program_transformer_tasks_total_set(pt_tasks_len.load(Ordering::Relaxed)); diff --git a/grpc-ingest/src/main.rs b/grpc-ingest/src/main.rs index 9ca97d111..25beb0ecd 100644 --- a/grpc-ingest/src/main.rs +++ b/grpc-ingest/src/main.rs @@ -71,14 +71,14 @@ async fn main() -> anyhow::Result<()> { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; - grpc::run(config).await + grpc::run_v2(config).await } ArgsAction::Ingester => { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; config.check(); - ingester::run(config).await + ingester::run_v2(config).await } ArgsAction::DownloadMetadata => { let config = config_load::(&args.config) diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index bd7ee591f..e81de4255 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -127,10 +127,10 @@ pub fn redis_xlen_set(stream: &str, len: usize) { .set(len as i64); } -pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: &usize) { +pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) { REDIS_XADD_STATUS .with_label_values(&[stream, if status.is_ok() { "success" } else { "failed" }]) - .inc_by(*delta as u64); + .inc_by(delta as u64); } pub fn redis_xack_inc(stream: &str, delta: usize) { diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 8cb599264..0b6dc7c18 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -499,4 +499,8 @@ impl TrackedPipeline { Err(_) => Err(counts), } } + + pub fn size(&self) -> usize { + self.counts.values().sum() + } } From a15163004cc856b0c26a386383856a5b15b2114f Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 12 Aug 2024 11:38:35 +0200 Subject: [PATCH 29/72] chore: handle metadata json within the ingest command --- core/src/metadata_json.rs | 30 ++++++- grpc-ingest/config-grpc2redis.yml | 2 - grpc-ingest/config-ingester.yml | 6 +- grpc-ingest/src/config.rs | 84 +++++++++++++------- grpc-ingest/src/grpc.rs | 16 ++-- grpc-ingest/src/ingester.rs | 125 ++++++++++++++++++++++++++---- grpc-ingest/src/redis.rs | 43 +++++----- 7 files changed, 224 insertions(+), 82 deletions(-) diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs index 584278843..eec14a46e 100644 --- a/core/src/metadata_json.rs +++ b/core/src/metadata_json.rs @@ -120,7 +120,7 @@ fn spawn_task( let asset_data_id = bs58::encode(download_metadata_info.asset_data_id.clone()).into_string(); - if let Err(e) = perform_metadata_json_task(client, pool, download_metadata_info).await { + if let Err(e) = perform_metadata_json_task(client, pool, &download_metadata_info).await { error!("Asset {} failed: {}", asset_data_id, e); } @@ -202,12 +202,12 @@ pub enum MetadataJsonTaskError { pub async fn perform_metadata_json_task( client: Client, pool: sqlx::PgPool, - download_metadata_info: DownloadMetadataInfo, + download_metadata_info: &DownloadMetadataInfo, ) -> Result { match fetch_metadata_json(client, &download_metadata_info.uri).await { Ok(metadata) => { let active_model = asset_data::ActiveModel { - id: Set(download_metadata_info.asset_data_id), + id: Set(download_metadata_info.asset_data_id.clone()), metadata: Set(metadata), reindex: Set(Some(false)), ..Default::default() @@ -222,3 +222,27 @@ pub async fn perform_metadata_json_task( Err(e) => Err(MetadataJsonTaskError::Fetch(e)), } } + +pub struct DownloadMetadata { + client: Client, + pool: sqlx::PgPool, +} + +impl DownloadMetadata { + pub const fn new(client: Client, pool: sqlx::PgPool) -> Self { + Self { client, pool } + } + + pub async fn handle_download( + &self, + download_metadata_info: &DownloadMetadataInfo, + ) -> Result<(), MetadataJsonTaskError> { + perform_metadata_json_task( + self.client.clone(), + self.pool.clone(), + download_metadata_info, + ) + .await + .map(|_| ()) + } +} diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index d763ca9bb..2362b1c61 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -5,7 +5,6 @@ commitment: finalized accounts: stream: ACCOUNTS stream_maxlen: 100_000_000 - stream_data_key: data filter: owner: - "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s" @@ -17,7 +16,6 @@ accounts: transactions: stream: TRANSACTIONS stream_maxlen: 10_000_000 - stream_data_key: data filter: account_include: - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 123d2e16f..c64902f4b 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -4,12 +4,13 @@ redis: group: ingester consumer: consumer # every new ingester instance should have uniq name streams: - - type: account # possible values: `account`, `metadata_json`, `transaction`, required for message decoding + - type: account # possible values: `account`, `metadatajson`, `transaction`, required for message decoding stream: ACCOUNTS data_key: data + - type: metadatajson + stream: METADATA_JSON - type: transaction stream: TRANSACTIONS - data_key: data xack_batch_max_size: 100 xack_batch_max_idle_ms: 10 xack_max_in_process: 100 @@ -22,7 +23,6 @@ postgres: min_connections: 10 max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible program_transformer: - transactions_cl_audits: false max_tasks_in_process: 40 download_metadata: max_attempts: 3 diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index a9cf25457..9fa98c1ca 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -12,7 +12,7 @@ use { pub const REDIS_STREAM_ACCOUNTS: &str = "ACCOUNTS"; pub const REDIS_STREAM_TRANSACTIONS: &str = "TRANSACTIONS"; -pub const REDIS_STREAM_METADETA_JSONS: &str = "METADETA_JSONS"; +pub const REDIS_STREAM_METADATA_JSON: &str = "METADATA_JSON"; pub const REDIS_STREAM_DATA_KEY: &str = "data"; pub async fn load(path: impl AsRef + Copy) -> anyhow::Result @@ -104,31 +104,20 @@ impl ConfigGrpcAccounts { #[derive(Debug, Deserialize)] pub struct ConfigGrpcTransactions { - #[serde(default = "ConfigGrpcTransactions::default_stream")] pub stream: String, #[serde( default = "ConfigGrpcTransactions::default_stream_maxlen", deserialize_with = "deserialize_usize_str" )] pub stream_maxlen: usize, - #[serde(default = "ConfigGrpcTransactions::default_stream_data_key")] - pub stream_data_key: String, pub filter: ConfigGrpcRequestTransactions, } impl ConfigGrpcTransactions { - pub fn default_stream() -> String { - REDIS_STREAM_TRANSACTIONS.to_owned() - } - pub const fn default_stream_maxlen() -> usize { 10_000_000 } - - pub fn default_stream_data_key() -> String { - REDIS_STREAM_DATA_KEY.to_owned() - } } #[derive(Debug, Deserialize)] @@ -243,11 +232,10 @@ impl ConfigIngesterRedis { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct ConfigIngesterRedisStream { pub stream_type: ConfigIngesterRedisStreamType, - pub stream: String, - pub data_key: String, + pub stream: &'static str, pub xack_batch_max_size: usize, pub xack_batch_max_idle: Duration, pub xack_max_in_process: usize, @@ -262,8 +250,6 @@ impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { struct Raw { #[serde(rename = "type")] pub stream_type: ConfigIngesterRedisStreamType, - pub stream: Option, - pub data_key: Option, #[serde( default = "default_xack_batch_max_size", deserialize_with = "deserialize_usize_str" @@ -295,18 +281,14 @@ impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { } let raw = Raw::deserialize(deserializer)?; + Ok(Self { stream_type: raw.stream_type, - stream: raw.stream.unwrap_or_else(|| match raw.stream_type { - ConfigIngesterRedisStreamType::Account => REDIS_STREAM_ACCOUNTS.to_owned(), - ConfigIngesterRedisStreamType::Transaction => REDIS_STREAM_TRANSACTIONS.to_owned(), - ConfigIngesterRedisStreamType::MetadataJson => { - REDIS_STREAM_METADETA_JSONS.to_owned() - } - }), - data_key: raw - .data_key - .unwrap_or_else(|| REDIS_STREAM_DATA_KEY.to_owned()), + stream: match raw.stream_type { + ConfigIngesterRedisStreamType::Account => REDIS_STREAM_ACCOUNTS, + ConfigIngesterRedisStreamType::Transaction => REDIS_STREAM_TRANSACTIONS, + ConfigIngesterRedisStreamType::MetadataJson => REDIS_STREAM_METADATA_JSON, + }, xack_batch_max_size: raw.xack_batch_max_size, xack_batch_max_idle: raw.xack_batch_max_idle, xack_max_in_process: raw.xack_max_in_process, @@ -362,19 +344,63 @@ impl ConfigIngesterProgramTransformer { } } -#[derive(Debug, Clone, Copy, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigIngesterDownloadMetadata { #[serde( default = "ConfigIngesterDownloadMetadata::default_max_attempts", deserialize_with = "deserialize_usize_str" )] pub max_attempts: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_request_timeout", + deserialize_with = "deserialize_duration_str", + rename = "request_timeout_ms" + )] + pub request_timeout: Duration, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_stream_maxlen", + deserialize_with = "deserialize_usize_str" + )] + pub stream_maxlen: usize, + #[serde(default = "ConfigIngesterDownloadMetadata::default_stream")] + pub stream: String, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_stream_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub pipeline_max_size: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_pipeline_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "pipeline_max_idle_ms" + )] + pub pipeline_max_idle: Duration, } impl ConfigIngesterDownloadMetadata { + pub const fn default_pipeline_max_idle() -> Duration { + Duration::from_millis(10) + } + + pub const fn default_stream_max_size() -> usize { + 10 + } + + pub fn default_stream() -> String { + REDIS_STREAM_METADATA_JSON.to_owned() + } + + pub const fn default_stream_maxlen() -> usize { + 10_000_000 + } + pub const fn default_max_attempts() -> usize { 3 } + + pub const fn default_request_timeout() -> Duration { + Duration::from_millis(3_000) + } } #[derive(Debug, Deserialize)] @@ -412,6 +438,8 @@ pub struct ConfigDownloadMetadataOpts { rename = "download_timeout_ms" )] pub download_timeout: Duration, + + pub stream: ConfigIngesterRedisStream, } impl ConfigDownloadMetadataOpts { diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 97ba2b406..8dd98f95f 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -1,6 +1,6 @@ use { crate::{ - config::ConfigGrpc, + config::{ConfigGrpc, REDIS_STREAM_DATA_KEY}, prom::redis_xadd_status_inc, redis::{metrics_xlen, TrackedPipeline}, util::create_shutdown, @@ -112,11 +112,8 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { GrpcJob::ProcessSubscribeUpdate(update) => { let accounts_stream = config.accounts.stream.clone(); let accounts_stream_maxlen = config.accounts.stream_maxlen; - let accounts_stream_data_key = config.accounts.stream_data_key.clone(); let transactions_stream = config.transactions.stream.clone(); let transactions_stream_maxlen = config.transactions.stream_maxlen; - let transactions_stream_data_key = - config.transactions.stream_data_key.clone(); let SubscribeUpdate { update_oneof, .. } = *update; @@ -129,7 +126,7 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { &accounts_stream, StreamMaxlen::Approx(accounts_stream_maxlen), "*", - &[(&accounts_stream_data_key, account.encode_to_vec())], + account.encode_to_vec(), ); debug!(message = "Account update", ?account,); @@ -139,10 +136,7 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { &transactions_stream, StreamMaxlen::Approx(transactions_stream_maxlen), "*", - &[( - &transactions_stream_data_key, - transaction.encode_to_vec(), - )], + transaction.encode_to_vec(), ); debug!(message = "Transaction update", ?transaction); @@ -275,7 +269,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { &config.accounts.stream, StreamMaxlen::Approx(config.accounts.stream_maxlen), "*", - &[(&config.accounts.stream_data_key, account.encode_to_vec())], + &[(REDIS_STREAM_DATA_KEY, account.encode_to_vec())], ); pipe_accounts += 1; @@ -285,7 +279,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { &config.transactions.stream, StreamMaxlen::Approx(config.transactions.stream_maxlen), "*", - &[(&config.transactions.stream_data_key, transaction.encode_to_vec())] + &[(REDIS_STREAM_DATA_KEY, transaction.encode_to_vec())] ); pipe_transactions += 1; diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index d79364ee3..22eb7897a 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,39 +1,51 @@ use { crate::{ config::{ConfigIngester, ConfigIngesterDownloadMetadata}, - download_metadata::TASK_TYPE, + download_metadata::{self, TASK_TYPE}, postgres::{create_pool as pg_create_pool, metrics_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, - program_transformer_tasks_total_set, ProgramTransformerTaskStatusKind, + program_transformer_tasks_total_set, redis_xadd_status_inc, + ProgramTransformerTaskStatusKind, + }, + redis::{ + metrics_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo, + TrackedPipeline, }, - redis::{metrics_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo}, util::create_shutdown, }, chrono::Utc, crypto::{digest::Digest, sha2::Sha256}, - das_core::{DownloadMetadataInfo, DownloadMetadataNotifier}, + das_core::{ + perform_metadata_json_task, DownloadMetadata, DownloadMetadataInfo, + DownloadMetadataNotifier, + }, digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, futures::{ future::{pending, BoxFuture, FusedFuture, FutureExt}, stream::StreamExt, + Future, }, opentelemetry_sdk::trace::Config, program_transformers::{error::ProgramTransformerError, ProgramTransformer}, + redis::{aio::MultiplexedConnection, streams::StreamMaxlen}, sea_orm::{ entity::{ActiveModelTrait, ActiveValue}, error::{DbErr, RuntimeErr}, SqlxPostgresConnector, }, + serde::Serialize, sqlx::{Error as SqlxError, PgPool}, std::{ borrow::Cow, + pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, Arc, }, }, tokio::{ + sync::Mutex, task::JoinSet, time::{sleep, Duration}, }, @@ -41,29 +53,77 @@ use { executor::{Executor, Nonblock, Tokio}, prelude::*, }, - tracing::{error, warn}, + tracing::{debug, error, warn}, }; enum IngestJob { SaveMessage(RedisStreamMessageInfo), + FlushRedisPipe(Arc>, MultiplexedConnection), +} + +fn download_metadata_notifier_v2( + pipe: Arc>, + stream: String, + stream_maxlen: usize, +) -> anyhow::Result { + Ok( + Box::new( + move |info: DownloadMetadataInfo| -> BoxFuture< + 'static, + Result<(), Box>, + > { + let pipe = Arc::clone(&pipe); + let stream = stream.clone(); + Box::pin(async move { + download_metadata_inserted_total_inc(); + + let mut pipe = pipe.lock().await; + let info_bytes = serde_json::to_vec(&info)?; + + pipe.xadd_maxlen( + &stream, + StreamMaxlen::Approx(stream_maxlen), + "*", + info_bytes, + ); + + Ok(()) + }) + }, + ), + ) } pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { - let client = redis::Client::open(config.redis.url.clone())?; - let connection = client.get_multiplexed_tokio_connection().await?; + let redis_client = redis::Client::open(config.redis.url.clone())?; + let connection = redis_client.get_multiplexed_tokio_connection().await?; let pool = pg_create_pool(config.postgres).await?; - let (mut redis_messages, redis_tasks_fut) = RedisStream::new(config.redis, connection).await?; + let pipe = Arc::new(Mutex::new(TrackedPipeline::default())); + let (mut redis_messages, redis_tasks_fut) = + RedisStream::new(config.redis, connection.clone()).await?; tokio::pin!(redis_tasks_fut); + let stream = config.download_metadata.stream.clone(); + let stream_maxlen = config.download_metadata.stream_maxlen; + + let accounts_download_metadata_notifier = + download_metadata_notifier_v2(Arc::clone(&pipe), stream.clone(), stream_maxlen)?; + let transactions_download_metadata_notifier = + download_metadata_notifier_v2(Arc::clone(&pipe), stream.clone(), stream_maxlen)?; + let pt_accounts = Arc::new(ProgramTransformer::new( pool.clone(), - create_download_metadata_notifier(pool.clone(), config.download_metadata)?, + accounts_download_metadata_notifier, )); let pt_transactions = Arc::new(ProgramTransformer::new( pool.clone(), - create_download_metadata_notifier(pool.clone(), config.download_metadata)?, + transactions_download_metadata_notifier, )); + let http_client = reqwest::Client::builder() + .timeout(config.download_metadata.request_timeout) + .build()?; + let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); let exec = Executor::builder(Nonblock(Tokio)) @@ -71,9 +131,25 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { .build(move |update, _handle| { let pt_accounts = Arc::clone(&pt_accounts); let pt_transactions = Arc::clone(&pt_transactions); + let download_metadata = Arc::clone(&download_metadata); async move { match update { + IngestJob::FlushRedisPipe(pipe, connection) => { + let mut pipe = pipe.lock().await; + let mut connection = connection; + + let flush = pipe.flush(&mut connection).await; + + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let counts = flush.as_ref().unwrap_or_else(|counts| counts); + + for (stream, count) in counts.iter() { + redis_xadd_status_inc(stream, status, *count); + } + + debug!(message = "Redis pipe flushed", ?status, ?counts); + } IngestJob::SaveMessage(msg) => { let result = match &msg.get_data() { ProgramTransformerInfo::Account(account) => { @@ -82,6 +158,14 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { ProgramTransformerInfo::Transaction(transaction) => { pt_transactions.handle_transaction(transaction).await } + ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + download_metadata + .handle_download(download_metadata_info) + .await + .map_err(|e| { + ProgramTransformerError::AssetIndexError(e.to_string()) + }) + } }; match result { Ok(()) => program_transformer_task_status_inc( @@ -118,7 +202,11 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { } } - let _ = msg.ack(); + if let Err(e) = msg.ack() { + error!("Failed to acknowledge message: {:?}", e); + } else { + debug!("Message acknowledged successfully"); + } () } @@ -130,6 +218,9 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { loop { tokio::select! { + _ = sleep(config.download_metadata.pipeline_max_idle) => { + exec.push(IngestJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); + } Some(msg) = redis_messages.recv() => { exec.push(IngestJob::SaveMessage(msg)); } @@ -168,8 +259,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { .redis .streams .iter() - .map(|config| config.stream.clone()) - .collect::>(); + .map(|config| config.stream.to_string()) + .collect::>(); async move { metrics_xlen(connection, &streams).await } }); tokio::pin!(jh_metrics_xlen); @@ -188,7 +279,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { // program transforms related let pt_accounts = Arc::new(ProgramTransformer::new( pgpool.clone(), - create_download_metadata_notifier(pgpool.clone(), config.download_metadata)?, + create_download_metadata_notifier(pgpool.clone(), config.download_metadata.clone())?, )); let pt_transactions = Arc::new(ProgramTransformer::new( pgpool.clone(), @@ -259,6 +350,9 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { ProgramTransformerInfo::Transaction(transaction) => { pt_transactions.handle_transaction(transaction).await } + ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + todo!() + } }; macro_rules! log_or_bail { @@ -275,6 +369,9 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { $error ) } + ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + todo!() + } } }; } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 0b6dc7c18..70fd585e6 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -1,8 +1,9 @@ use { crate::{ - config::{ConfigIngesterRedis, ConfigIngesterRedisStreamType}, + config::{ConfigIngesterRedis, ConfigIngesterRedisStreamType, REDIS_STREAM_DATA_KEY}, prom::{redis_xack_inc, redis_xlen_set}, }, + das_core::DownloadMetadataInfo, futures::future::{BoxFuture, Fuse, FutureExt}, program_transformers::{AccountInfo, TransactionInfo}, redis::{ @@ -13,6 +14,7 @@ use { }, AsyncCommands, Cmd, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, + serde::{Deserialize, Serialize}, solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{ collections::HashMap, @@ -89,7 +91,6 @@ struct RedisStreamInfo { consumer: String, stream_name: String, stream_type: ConfigIngesterRedisStreamType, - stream_data_key: String, xack_batch_max_size: usize, xack_batch_max_idle: Duration, xack_max_in_process: usize, @@ -99,6 +100,7 @@ struct RedisStreamInfo { pub enum ProgramTransformerInfo { Account(AccountInfo), Transaction(TransactionInfo), + MetadataJson(DownloadMetadataInfo), } #[derive(Debug)] @@ -116,7 +118,7 @@ impl RedisStreamMessageInfo { ) -> anyhow::Result { let to_anyhow = |error: String| anyhow::anyhow!(error); - let data = match map.get(&stream.stream_data_key) { + let data = match map.get(REDIS_STREAM_DATA_KEY) { Some(RedisValue::Data(vec)) => match stream.stream_type { ConfigIngesterRedisStreamType::Account => { let SubscribeUpdateAccount { account, slot, .. } = @@ -177,16 +179,20 @@ impl RedisStreamMessageInfo { .map_err(to_anyhow)?, }) } - ConfigIngesterRedisStreamType::MetadataJson => todo!(), + ConfigIngesterRedisStreamType::MetadataJson => { + let info: DownloadMetadataInfo = serde_json::from_slice(vec.as_ref())?; + + ProgramTransformerInfo::MetadataJson(info) + } }, Some(_) => anyhow::bail!( "invalid data (key: {:?}) from stream {:?}", - stream.stream_data_key, + REDIS_STREAM_DATA_KEY, stream.stream_name ), None => anyhow::bail!( "failed to get data (key: {:?}) from stream {:?}", - stream.stream_data_key, + REDIS_STREAM_DATA_KEY, stream.stream_name ), }; @@ -219,7 +225,7 @@ impl RedisStream { for stream in config.streams.iter() { xgroup_create( &mut connection, - &stream.stream, + stream.stream, &config.group, &config.consumer, ) @@ -239,15 +245,14 @@ impl RedisStream { let info = Arc::new(RedisStreamInfo { group: config.group.clone(), consumer: config.consumer.clone(), - stream_name: stream.stream.clone(), + stream_name: stream.stream.to_string(), stream_type: stream.stream_type, - stream_data_key: stream.data_key.clone(), xack_batch_max_size: stream.xack_batch_max_size, xack_batch_max_idle: stream.xack_batch_max_idle, xack_max_in_process: stream.xack_max_in_process, }); ack_tasks.push((Arc::clone(&info), ack_rx)); - (stream.stream.clone(), (ack_tx, info)) + (stream.stream, (ack_tx, info)) }) .collect::>(); @@ -296,7 +301,7 @@ impl RedisStream { async fn run_prefetch( config: ConfigIngesterRedis, - streams: HashMap, Arc)>, + streams: HashMap<&str, (mpsc::UnboundedSender, Arc)>, mut connection: MultiplexedConnection, messages_tx: mpsc::Sender, shutdown: Arc, @@ -358,7 +363,7 @@ impl RedisStream { return Ok(()); } - let streams_keys = streams.keys().map(|name| name.as_str()).collect::>(); + let streams_keys = streams.keys().collect::>(); let streams_ids = (0..streams_keys.len()).map(|_| ">").collect::>(); while !shutdown.load(Ordering::Relaxed) { let opts = StreamReadOptions::default() @@ -373,7 +378,7 @@ impl RedisStream { } for StreamKey { key, ids } in results.keys { - let (ack_tx, stream) = match streams.get(&key) { + let (ack_tx, stream) = match streams.get(key.as_str()) { Some(value) => value, None => anyhow::bail!("unknown stream: {:?}", key), }; @@ -471,17 +476,13 @@ impl Default for TrackedPipeline { type TrackedStreamCounts = HashMap; impl TrackedPipeline { - pub fn xadd_maxlen( - &mut self, - key: &str, - maxlen: StreamMaxlen, - id: F, - fields: &[(&str, V)], - ) where + pub fn xadd_maxlen(&mut self, key: &str, maxlen: StreamMaxlen, id: F, field: V) + where F: redis::ToRedisArgs, V: redis::ToRedisArgs, { - self.pipeline.xadd_maxlen(key, maxlen, id, fields); + self.pipeline + .xadd_maxlen(key, maxlen, id, &[(REDIS_STREAM_DATA_KEY, field)]); *self.counts.entry(key.to_string()).or_insert(0) += 1; } From f398880193932976c9a11bc739bf045890ce26d1 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 12 Aug 2024 13:38:12 +0200 Subject: [PATCH 30/72] test: metadata json stream processing --- grpc-ingest/config-ingester.yml | 7 +++---- grpc-ingest/src/grpc.rs | 13 +++---------- grpc-ingest/src/ingester.rs | 15 ++++++--------- grpc-ingest/src/redis.rs | 3 +-- program_transformers/src/lib.rs | 1 - 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index c64902f4b..f8b278be1 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -4,16 +4,15 @@ redis: group: ingester consumer: consumer # every new ingester instance should have uniq name streams: - - type: account # possible values: `account`, `metadatajson`, `transaction`, required for message decoding + - type: account stream: ACCOUNTS - data_key: data - - type: metadatajson - stream: METADATA_JSON - type: transaction stream: TRANSACTIONS xack_batch_max_size: 100 xack_batch_max_idle_ms: 10 xack_max_in_process: 100 + - type: metadatajson + stream: METADATA_JSON prefetch_queue_size: 1_000 # max number of messages available in the read queue for processing xpending_max: 250 # used for reading pending messages xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 8dd98f95f..00ad182f4 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -7,15 +7,8 @@ use { }, anyhow::Context, futures::{channel::mpsc, stream::StreamExt, SinkExt}, - lru::LruCache, - opentelemetry_sdk::trace::Config, - redis::{ - aio::MultiplexedConnection, streams::StreamMaxlen, Pipeline, RedisResult, - Value as RedisValue, - }, - serde::de, - sqlx::pool, - std::{collections::HashMap, num::NonZeroUsize, sync::Arc, time::Duration}, + redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, + std::{collections::HashMap, sync::Arc, time::Duration}, tokio::{ spawn, sync::Mutex, @@ -27,7 +20,6 @@ use { prelude::*, }, tracing::{debug, warn}, - tracing_subscriber::field::debug, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::{SubscribeRequest, SubscribeUpdate}, @@ -176,6 +168,7 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { Ok(()) } +#[allow(dead_code)] pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let config = Arc::new(config); let (tx, mut rx) = mpsc::channel::(config.geyser_update_message_buffer_size); // Adjust buffer size as needed diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 22eb7897a..ba8c125f0 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,7 +1,7 @@ use { crate::{ config::{ConfigIngester, ConfigIngesterDownloadMetadata}, - download_metadata::{self, TASK_TYPE}, + download_metadata::{TASK_TYPE}, postgres::{create_pool as pg_create_pool, metrics_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, @@ -17,16 +17,14 @@ use { chrono::Utc, crypto::{digest::Digest, sha2::Sha256}, das_core::{ - perform_metadata_json_task, DownloadMetadata, DownloadMetadataInfo, + DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier, }, digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, futures::{ future::{pending, BoxFuture, FusedFuture, FutureExt}, stream::StreamExt, - Future, }, - opentelemetry_sdk::trace::Config, program_transformers::{error::ProgramTransformerError, ProgramTransformer}, redis::{aio::MultiplexedConnection, streams::StreamMaxlen}, sea_orm::{ @@ -34,11 +32,9 @@ use { error::{DbErr, RuntimeErr}, SqlxPostgresConnector, }, - serde::Serialize, sqlx::{Error as SqlxError, PgPool}, std::{ borrow::Cow, - pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -208,7 +204,7 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { debug!("Message acknowledged successfully"); } - () + } } } @@ -226,6 +222,7 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { } Some(signal) = shutdown.next() => { warn!("{signal} received, waiting spawned tasks..."); + exec.push(IngestJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); break; } result = &mut redis_tasks_fut => { @@ -350,7 +347,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { ProgramTransformerInfo::Transaction(transaction) => { pt_transactions.handle_transaction(transaction).await } - ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + ProgramTransformerInfo::MetadataJson(_download_metadata_info) => { todo!() } }; @@ -369,7 +366,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { $error ) } - ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + ProgramTransformerInfo::MetadataJson(_download_metadata_info) => { todo!() } } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 70fd585e6..85b239fb4 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -12,9 +12,8 @@ use { StreamClaimReply, StreamId, StreamKey, StreamMaxlen, StreamPendingCountReply, StreamReadOptions, StreamReadReply, }, - AsyncCommands, Cmd, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, + AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, - serde::{Deserialize, Serialize}, solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{ collections::HashMap, diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index bef34b9ab..aada07fc0 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -20,7 +20,6 @@ use { entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, }, - serde::{Deserialize, Serialize}, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, sqlx::PgPool, From 70eef04644786ec09863dfa54394f37b11515dad Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 14 Aug 2024 10:00:55 +0200 Subject: [PATCH 31/72] feat: set num threads for topograph. report on postgres and redis in ingester using dedicate exec --- grpc-ingest/config-grpc2redis.yml | 2 + grpc-ingest/config-ingester.yml | 4 +- grpc-ingest/src/config.rs | 49 +++++-- grpc-ingest/src/grpc.rs | 119 ++++++++-------- grpc-ingest/src/ingester.rs | 223 ++++++++++++++++++------------ grpc-ingest/src/postgres.rs | 8 +- grpc-ingest/src/redis.rs | 16 +++ 7 files changed, 252 insertions(+), 169 deletions(-) diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index 2362b1c61..c4d0067af 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -2,6 +2,8 @@ prometheus: 127.0.0.1:8873 geyser_endpoint: http://127.0.0.1:10000 x_token: null commitment: finalized +topograph: + num_threads: 5 accounts: stream: ACCOUNTS stream_maxlen: 100_000_000 diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index f8b278be1..16869a27e 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -1,4 +1,6 @@ prometheus: 127.0.0.1:8874 +topograph: + num_threads: 1 redis: url: redis://localhost:6379 group: ingester @@ -22,6 +24,6 @@ postgres: min_connections: 10 max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible program_transformer: - max_tasks_in_process: 40 + max_tasks_in_process: 100 download_metadata: max_attempts: 3 diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 9fa98c1ca..c9f243418 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -32,13 +32,26 @@ where } } -#[derive(Debug, Default, Deserialize)] +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(default)] +pub struct ConfigTopograph { + #[serde(default = "ConfigTopograph::default_num_threads")] + pub num_threads: usize, +} + +impl ConfigTopograph { + pub const fn default_num_threads() -> usize { + 5 + } +} + +#[derive(Debug, Clone, Default, Deserialize)] #[serde(default)] pub struct ConfigPrometheus { pub prometheus: Option, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpc { pub x_token: Option, @@ -61,6 +74,8 @@ pub struct ConfigGrpc { pub solana_seen_event_cache_max_size: usize, pub redis: ConfigGrpcRedis, + + pub topograph: ConfigTopograph, } impl ConfigGrpc { @@ -73,7 +88,7 @@ impl ConfigGrpc { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpcAccounts { #[serde(default = "ConfigGrpcAccounts::default_stream")] pub stream: String, @@ -102,7 +117,7 @@ impl ConfigGrpcAccounts { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpcTransactions { pub stream: String, #[serde( @@ -120,7 +135,7 @@ impl ConfigGrpcTransactions { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpcRedis { pub url: String, #[serde( @@ -163,23 +178,27 @@ where Ok(Duration::from_millis(ms as u64)) } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigIngester { pub redis: ConfigIngesterRedis, pub postgres: ConfigIngesterPostgres, - pub program_transformer: ConfigIngesterProgramTransformer, pub download_metadata: ConfigIngesterDownloadMetadata, + pub topograph: ConfigTopograph, + pub program_transformer: ConfigIngesterProgramTransformer, } impl ConfigIngester { pub fn check(&self) { - if self.postgres.max_connections < self.program_transformer.max_tasks_in_process { - warn!("`postgres.max_connections` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible"); + if self.postgres.max_connections < self.topograph.num_threads { + warn!( + "postgres.max_connections ({}) should be more than the number of threads ({})", + self.postgres.max_connections, self.topograph.num_threads + ); } } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigIngesterRedis { pub url: String, #[serde(default = "ConfigIngesterRedis::default_group")] @@ -246,7 +265,7 @@ impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { where D: de::Deserializer<'de>, { - #[derive(Debug, Deserialize)] + #[derive(Debug, Clone, Copy, Deserialize)] struct Raw { #[serde(rename = "type")] pub stream_type: ConfigIngesterRedisStreamType, @@ -296,7 +315,7 @@ impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { } } -#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ConfigIngesterRedisStreamType { Account, @@ -304,7 +323,7 @@ pub enum ConfigIngesterRedisStreamType { MetadataJson, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigIngesterPostgres { pub url: String, #[serde( @@ -329,7 +348,7 @@ impl ConfigIngesterPostgres { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigIngesterProgramTransformer { #[serde( default = "ConfigIngesterProgramTransformer::default_max_tasks_in_process", @@ -403,7 +422,7 @@ impl ConfigIngesterDownloadMetadata { } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct ConfigDownloadMetadata { pub postgres: ConfigIngesterPostgres, pub download_metadata: ConfigDownloadMetadataOpts, diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 00ad182f4..002b52cb9 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -76,74 +76,73 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { let pool_connection = connection.clone(); let pool_pipe = Arc::clone(&pipe); - let exec = - Executor::builder(Nonblock(Tokio)) - .num_threads(None) - .build(move |update, _handle| { - let config = Arc::clone(&pool_config); - let connection = pool_connection.clone(); - let pipe = Arc::clone(&pool_pipe); - - async move { - match update { - GrpcJob::FlushRedisPipe => { - let mut pipe = pipe.lock().await; - let mut connection = connection; - - let flush = pipe.flush(&mut connection).await; - - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); - - for (stream, count) in counts.iter() { - redis_xadd_status_inc(stream, status, *count); - } + let exec = Executor::builder(Nonblock(Tokio)) + .num_threads(Some(config.topograph.num_threads)) + .build(move |update, _handle| { + let config = Arc::clone(&pool_config); + let connection = pool_connection.clone(); + let pipe = Arc::clone(&pool_pipe); + + async move { + match update { + GrpcJob::FlushRedisPipe => { + let mut pipe = pipe.lock().await; + let mut connection = connection; + + let flush = pipe.flush(&mut connection).await; - debug!(message = "Redis pipe flushed", ?status, ?counts); + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let counts = flush.as_ref().unwrap_or_else(|counts| counts); + + for (stream, count) in counts.iter() { + redis_xadd_status_inc(stream, status, *count); } - GrpcJob::ProcessSubscribeUpdate(update) => { - let accounts_stream = config.accounts.stream.clone(); - let accounts_stream_maxlen = config.accounts.stream_maxlen; - let transactions_stream = config.transactions.stream.clone(); - let transactions_stream_maxlen = config.transactions.stream_maxlen; - - let SubscribeUpdate { update_oneof, .. } = *update; - - let mut pipe = pipe.lock().await; - - if let Some(update) = update_oneof { - match update { - UpdateOneof::Account(account) => { - pipe.xadd_maxlen( - &accounts_stream, - StreamMaxlen::Approx(accounts_stream_maxlen), - "*", - account.encode_to_vec(), - ); - - debug!(message = "Account update", ?account,); - } - UpdateOneof::Transaction(transaction) => { - pipe.xadd_maxlen( - &transactions_stream, - StreamMaxlen::Approx(transactions_stream_maxlen), - "*", - transaction.encode_to_vec(), - ); - - debug!(message = "Transaction update", ?transaction); - } - var => warn!(message = "Unknown update variant", ?var), + + debug!(message = "Redis pipe flushed", ?status, ?counts); + } + GrpcJob::ProcessSubscribeUpdate(update) => { + let accounts_stream = config.accounts.stream.clone(); + let accounts_stream_maxlen = config.accounts.stream_maxlen; + let transactions_stream = config.transactions.stream.clone(); + let transactions_stream_maxlen = config.transactions.stream_maxlen; + + let SubscribeUpdate { update_oneof, .. } = *update; + + let mut pipe = pipe.lock().await; + + if let Some(update) = update_oneof { + match update { + UpdateOneof::Account(account) => { + pipe.xadd_maxlen( + &accounts_stream, + StreamMaxlen::Approx(accounts_stream_maxlen), + "*", + account.encode_to_vec(), + ); + + debug!(message = "Account update", ?account,); + } + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &transactions_stream, + StreamMaxlen::Approx(transactions_stream_maxlen), + "*", + transaction.encode_to_vec(), + ); + + debug!(message = "Transaction update", ?transaction); } + var => warn!(message = "Unknown update variant", ?var), } + } - if pipe.size() >= config.redis.pipeline_max_size { - // handle.push(GrpcJob::FlushRedisPipe); - } + if pipe.size() >= config.redis.pipeline_max_size { + // handle.push(GrpcJob::FlushRedisPipe); } } } - })?; + } + })?; let deadline_config = Arc::clone(&config); diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index ba8c125f0..17ce8c414 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,25 +1,22 @@ use { crate::{ config::{ConfigIngester, ConfigIngesterDownloadMetadata}, - download_metadata::{TASK_TYPE}, - postgres::{create_pool as pg_create_pool, metrics_pgpool}, + download_metadata::TASK_TYPE, + postgres::{create_pool as pg_create_pool, metrics_pgpool, report_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, program_transformer_tasks_total_set, redis_xadd_status_inc, ProgramTransformerTaskStatusKind, }, redis::{ - metrics_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo, + metrics_xlen, report_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo, TrackedPipeline, }, util::create_shutdown, }, chrono::Utc, crypto::{digest::Digest, sha2::Sha256}, - das_core::{ - DownloadMetadata, DownloadMetadataInfo, - DownloadMetadataNotifier, - }, + das_core::{DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier}, digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, futures::{ future::{pending, BoxFuture, FusedFuture, FutureExt}, @@ -57,6 +54,11 @@ enum IngestJob { FlushRedisPipe(Arc>, MultiplexedConnection), } +enum ReportJob { + Redis(Vec, MultiplexedConnection), + Postgres(PgPool), +} + fn download_metadata_notifier_v2( pipe: Arc>, stream: String, @@ -97,16 +99,29 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { let pipe = Arc::new(Mutex::new(TrackedPipeline::default())); let (mut redis_messages, redis_tasks_fut) = - RedisStream::new(config.redis, connection.clone()).await?; + RedisStream::new(config.redis.clone(), connection.clone()).await?; tokio::pin!(redis_tasks_fut); - let stream = config.download_metadata.stream.clone(); - let stream_maxlen = config.download_metadata.stream_maxlen; - - let accounts_download_metadata_notifier = - download_metadata_notifier_v2(Arc::clone(&pipe), stream.clone(), stream_maxlen)?; - let transactions_download_metadata_notifier = - download_metadata_notifier_v2(Arc::clone(&pipe), stream.clone(), stream_maxlen)?; + let streams = config + .redis + .streams + .iter() + .map(|config| config.stream.to_string()) + .collect::>(); + + let download_metadata_stream = config.download_metadata.stream.clone(); + let download_metadata_stream_maxlen = config.download_metadata.stream_maxlen; + + let accounts_download_metadata_notifier = download_metadata_notifier_v2( + Arc::clone(&pipe), + download_metadata_stream.clone(), + download_metadata_stream_maxlen, + )?; + let transactions_download_metadata_notifier = download_metadata_notifier_v2( + Arc::clone(&pipe), + download_metadata_stream.clone(), + download_metadata_stream_maxlen, + )?; let pt_accounts = Arc::new(ProgramTransformer::new( pool.clone(), @@ -121,99 +136,120 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { .build()?; let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); - let exec = - Executor::builder(Nonblock(Tokio)) - .num_threads(None) - .build(move |update, _handle| { - let pt_accounts = Arc::clone(&pt_accounts); - let pt_transactions = Arc::clone(&pt_transactions); - let download_metadata = Arc::clone(&download_metadata); + let exec = Executor::builder(Nonblock(Tokio)) + .num_threads(Some(config.topograph.num_threads)) + .build(move |update, _handle| { + let pt_accounts = Arc::clone(&pt_accounts); + let pt_transactions = Arc::clone(&pt_transactions); + let download_metadata = Arc::clone(&download_metadata); - async move { - match update { - IngestJob::FlushRedisPipe(pipe, connection) => { - let mut pipe = pipe.lock().await; - let mut connection = connection; + async move { + match update { + IngestJob::FlushRedisPipe(pipe, connection) => { + let mut pipe = pipe.lock().await; + let mut connection = connection; - let flush = pipe.flush(&mut connection).await; + let flush = pipe.flush(&mut connection).await; - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let counts = flush.as_ref().unwrap_or_else(|counts| counts); - for (stream, count) in counts.iter() { - redis_xadd_status_inc(stream, status, *count); + for (stream, count) in counts.iter() { + redis_xadd_status_inc(stream, status, *count); + } + + debug!(message = "Redis pipe flushed", ?status, ?counts); + } + IngestJob::SaveMessage(msg) => { + let result = match &msg.get_data() { + ProgramTransformerInfo::Account(account) => { + pt_accounts.handle_account_update(account).await + } + ProgramTransformerInfo::Transaction(transaction) => { + pt_transactions.handle_transaction(transaction).await + } + ProgramTransformerInfo::MetadataJson(download_metadata_info) => { + download_metadata + .handle_download(download_metadata_info) + .await + .map_err(|e| { + ProgramTransformerError::AssetIndexError(e.to_string()) + }) } + }; + match result { + Ok(()) => program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::Success, + ), + Err(ProgramTransformerError::NotImplemented) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::NotImplemented, + ); + error!("not implemented") + } + Err(ProgramTransformerError::DeserializationError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::DeserializationError, + ); - debug!(message = "Redis pipe flushed", ?status, ?counts); - } - IngestJob::SaveMessage(msg) => { - let result = match &msg.get_data() { - ProgramTransformerInfo::Account(account) => { - pt_accounts.handle_account_update(account).await - } - ProgramTransformerInfo::Transaction(transaction) => { - pt_transactions.handle_transaction(transaction).await - } - ProgramTransformerInfo::MetadataJson(download_metadata_info) => { - download_metadata - .handle_download(download_metadata_info) - .await - .map_err(|e| { - ProgramTransformerError::AssetIndexError(e.to_string()) - }) - } - }; - match result { - Ok(()) => program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::Success, - ), - Err(ProgramTransformerError::NotImplemented) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::NotImplemented, - ); - error!("not implemented") - } - Err(ProgramTransformerError::DeserializationError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::DeserializationError, - ); - - error!("failed to deserialize {:?}", error) - } - Err(ProgramTransformerError::ParsingError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::ParsingError, - ); - - error!("failed to parse {:?}", error) - } - Err(ProgramTransformerError::DatabaseError(error)) => { - error!("database error for {:?}", error) - } - Err(ProgramTransformerError::AssetIndexError(error)) => { - error!("indexing error for {:?}", error) - } - Err(error) => { - error!("failed to handle {:?}", error) - } + error!("failed to deserialize {:?}", error) } + Err(ProgramTransformerError::ParsingError(error)) => { + program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::ParsingError, + ); - if let Err(e) = msg.ack() { - error!("Failed to acknowledge message: {:?}", e); - } else { - debug!("Message acknowledged successfully"); + error!("failed to parse {:?}", error) } + Err(ProgramTransformerError::DatabaseError(error)) => { + error!("database error for {:?}", error) + } + Err(ProgramTransformerError::AssetIndexError(error)) => { + error!("indexing error for {:?}", error) + } + Err(error) => { + error!("failed to handle {:?}", error) + } + } - + if let Err(e) = msg.ack() { + error!("Failed to acknowledge message: {:?}", e); + } else { + debug!("Message acknowledged successfully"); } } } - })?; + } + })?; + + let reporting_exec = Executor::builder(Nonblock(Tokio)) + .num_threads(Some(1)) + .build(move |update, _handle| async move { + match update { + ReportJob::Postgres(pool) => { + report_pgpool(pool); + + debug!("Successfully reported Postgres pool metrics"); + } + ReportJob::Redis(streams, connection) => { + match report_xlen(connection, streams).await { + Ok(_) => debug!("Successfully reported Redis stream length"), + Err(e) => error!("Failed to report Redis stream length: {:?}", e), + } + } + } + })?; let mut shutdown = create_shutdown()?; loop { tokio::select! { + _ = sleep(Duration::from_millis(100)) => { + reporting_exec.push(ReportJob::Postgres(pool.clone())); + } + _ = sleep(Duration::from_millis(100)) => { + reporting_exec.push(ReportJob::Redis(streams.clone(), connection.clone())); + } _ = sleep(config.download_metadata.pipeline_max_idle) => { exec.push(IngestJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); } @@ -222,7 +258,6 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { } Some(signal) = shutdown.next() => { warn!("{signal} received, waiting spawned tasks..."); - exec.push(IngestJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); break; } result = &mut redis_tasks_fut => { @@ -234,9 +269,15 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { } } + exec.push(IngestJob::FlushRedisPipe( + Arc::clone(&pipe), + connection.clone(), + )); + redis_messages.shutdown(); exec.join_async().await; + reporting_exec.join_async().await; pool.close().await; diff --git a/grpc-ingest/src/postgres.rs b/grpc-ingest/src/postgres.rs index 05e535b4d..dd29ca730 100644 --- a/grpc-ingest/src/postgres.rs +++ b/grpc-ingest/src/postgres.rs @@ -20,10 +20,14 @@ pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result( + mut connection: C, + streams: Vec, +) -> anyhow::Result<()> { + let mut pipe = redis::pipe(); + for stream in &streams { + pipe.xlen(stream); + } + let xlens: Vec = pipe.query_async(&mut connection).await?; + + for (stream, xlen) in streams.iter().zip(xlens.into_iter()) { + redis_xlen_set(stream, xlen); + } + + Ok(()) +} pub async fn metrics_xlen( mut connection: C, streams: &[String], From 77a98f64326852037fe91464be98742d95e6dd85 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Tue, 20 Aug 2024 18:52:07 +0200 Subject: [PATCH 32/72] chore: switch to topograph for thread management. create separate ingest stream which handle reading messages and handling messages. --- Cargo.lock | 3 +- Cargo.toml | 2 +- grpc-ingest/Cargo.toml | 1 + grpc-ingest/config-grpc2redis.yml | 3 +- grpc-ingest/config-ingester.yml | 25 +- grpc-ingest/src/config.rs | 118 +++++- grpc-ingest/src/grpc.rs | 158 ++++---- grpc-ingest/src/ingester.rs | 287 +++++---------- grpc-ingest/src/prom.rs | 103 +++++- grpc-ingest/src/redis.rs | 577 +++++++++++++++++++++++++++++- program_transformers/src/lib.rs | 5 +- 11 files changed, 986 insertions(+), 296 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0384d59b..396e8e69c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1883,6 +1883,7 @@ dependencies = [ "serde_yaml", "solana-sdk", "sqlx", + "thiserror", "tokio", "topograph", "tracing", @@ -7806,7 +7807,7 @@ dependencies = [ [[package]] name = "topograph" version = "0.4.0" -source = "git+https://github.com/ray-kast/topograph?branch=v0.4-wip#f56fe34a5614507c50721e9555b412f18db0c51e" +source = "git+https://github.com/ray-kast/topograph?branch=v0.4-wip#c91c01c25f605c4bc4461e9b9cbf95d8e4038972" dependencies = [ "crossbeam", "dispose", diff --git a/Cargo.toml b/Cargo.toml index 8346aa30f..ffeffdf61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,7 +132,7 @@ tokio = "1.30.0" tokio-stream = "0.1.14" topograph = { version = "0.4.0", features = [ "tokio", -], git = "https://github.com/ray-kast/topograph", branch = "v0.4-wip" } +], git = "https://github.com/ray-kast/topograph", sha = "5449d34ff726c2ab9f4d3a59a7070ae7bdc76027", branch = "v0.4-wip" } tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" diff --git a/grpc-ingest/Cargo.toml b/grpc-ingest/Cargo.toml index 3da16c1e9..b4237fdc2 100644 --- a/grpc-ingest/Cargo.toml +++ b/grpc-ingest/Cargo.toml @@ -19,6 +19,7 @@ sqlx = { workspace = true, features = [ ] } chrono = { workspace = true } clap = { workspace = true, features = ["cargo", "derive"] } +thiserror = { workspace = true } das-core = { workspace = true } digital_asset_types = { workspace = true } futures = { workspace = true } diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index c4d0067af..0a7fef2d0 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -2,8 +2,7 @@ prometheus: 127.0.0.1:8873 geyser_endpoint: http://127.0.0.1:10000 x_token: null commitment: finalized -topograph: - num_threads: 5 +max_concurrency: 5 accounts: stream: ACCOUNTS stream_maxlen: 100_000_000 diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 16869a27e..56cd50b10 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -1,22 +1,11 @@ -prometheus: 127.0.0.1:8874 -topograph: - num_threads: 1 +prometheus: 127.0.0.1:8875 redis: url: redis://localhost:6379 group: ingester consumer: consumer # every new ingester instance should have uniq name - streams: - - type: account - stream: ACCOUNTS - - type: transaction - stream: TRANSACTIONS - xack_batch_max_size: 100 - xack_batch_max_idle_ms: 10 - xack_max_in_process: 100 - - type: metadatajson - stream: METADATA_JSON - prefetch_queue_size: 1_000 # max number of messages available in the read queue for processing - xpending_max: 250 # used for reading pending messages + streams: [] + prefetch_queue_size: 5_000 # max number of messages available in the read queue for processing + xpending_max: 1_000 # used for reading pending messages xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) xreadgroup_max: 1_000 # applied per each stream in one request postgres: @@ -25,5 +14,11 @@ postgres: max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible program_transformer: max_tasks_in_process: 100 +accounts: + name: ACCOUNTS +transactions: + name: TRANSACTIONS download_metadata: max_attempts: 3 + stream_config: + name: METADATA_JSON diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index c9f243418..a27781e9d 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -32,6 +32,64 @@ where } } +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigIngestStream { + pub name: String, + #[serde(default = "ConfigIngestStream::default_group")] + pub group: String, + #[serde(default = "ConfigIngestStream::default_consumer")] + pub consumer: String, + #[serde(default = "ConfigIngestStream::default_xack_batch_max_size")] + pub xack_batch_max_size: usize, + #[serde( + default = "ConfigIngestStream::default_xack_batch_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "xack_batch_max_idle_ms" + )] + pub xack_batch_max_idle: Duration, + #[serde(default = "ConfigIngestStream::default_batch_size")] + pub batch_size: usize, + #[serde( + default = "ConfigIngestStream::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, + #[serde( + default = "ConfigIngestStream::default_xack_buffer_size", + deserialize_with = "deserialize_usize_str" + )] + pub xack_buffer_size: usize, +} + +impl ConfigIngestStream { + pub const fn default_xack_buffer_size() -> usize { + 1_000 + } + pub const fn default_max_concurrency() -> usize { + 2 + } + + pub const fn default_xack_batch_max_idle() -> Duration { + Duration::from_millis(100) + } + + pub fn default_group() -> String { + "ingester".to_owned() + } + + pub fn default_consumer() -> String { + "consumer".to_owned() + } + + pub const fn default_xack_batch_max_size() -> usize { + 100 + } + + pub const fn default_batch_size() -> usize { + 100 + } +} + #[derive(Debug, Clone, Deserialize, Default)] #[serde(default)] pub struct ConfigTopograph { @@ -75,10 +133,18 @@ pub struct ConfigGrpc { pub redis: ConfigGrpcRedis, - pub topograph: ConfigTopograph, + #[serde( + default = "ConfigGrpc::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, } impl ConfigGrpc { + pub const fn default_max_concurrency() -> usize { + 10 + } + pub const fn default_geyser_update_message_buffer_size() -> usize { 100_000 } @@ -183,16 +249,21 @@ pub struct ConfigIngester { pub redis: ConfigIngesterRedis, pub postgres: ConfigIngesterPostgres, pub download_metadata: ConfigIngesterDownloadMetadata, - pub topograph: ConfigTopograph, pub program_transformer: ConfigIngesterProgramTransformer, + pub accounts: ConfigIngestStream, + pub transactions: ConfigIngestStream, } impl ConfigIngester { pub fn check(&self) { - if self.postgres.max_connections < self.topograph.num_threads { + let total_threads = self.program_transformer.account_num_threads + + self.program_transformer.transaction_num_threads + + self.program_transformer.metadata_json_num_threads; + + if self.postgres.max_connections < total_threads { warn!( "postgres.max_connections ({}) should be more than the number of threads ({})", - self.postgres.max_connections, self.topograph.num_threads + self.postgres.max_connections, total_threads ); } } @@ -355,16 +426,49 @@ pub struct ConfigIngesterProgramTransformer { deserialize_with = "deserialize_usize_str" )] pub max_tasks_in_process: usize, + #[serde( + default = "ConfigIngesterProgramTransformer::default_account_num_threads", + deserialize_with = "deserialize_usize_str" + )] + pub account_num_threads: usize, + #[serde( + default = "ConfigIngesterProgramTransformer::default_transaction_num_threads", + deserialize_with = "deserialize_usize_str" + )] + pub transaction_num_threads: usize, + #[serde( + default = "ConfigIngesterProgramTransformer::default_metadata_json_num_threads", + deserialize_with = "deserialize_usize_str" + )] + pub metadata_json_num_threads: usize, } impl ConfigIngesterProgramTransformer { + pub const fn default_account_num_threads() -> usize { + 5 + } + + pub const fn default_transaction_num_threads() -> usize { + 1 + } + + pub const fn default_metadata_json_num_threads() -> usize { + 1 + } + pub const fn default_max_tasks_in_process() -> usize { 40 } } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Default, Deserialize)] pub struct ConfigIngesterDownloadMetadata { + pub stream_config: ConfigIngestStream, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_num_threads", + deserialize_with = "deserialize_usize_str" + )] + pub num_threads: usize, #[serde( default = "ConfigIngesterDownloadMetadata::default_max_attempts", deserialize_with = "deserialize_usize_str" @@ -397,6 +501,10 @@ pub struct ConfigIngesterDownloadMetadata { } impl ConfigIngesterDownloadMetadata { + pub const fn default_num_threads() -> usize { + 2 + } + pub const fn default_pipeline_max_idle() -> Duration { Duration::from_millis(10) } diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 002b52cb9..41d94573d 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -18,6 +18,7 @@ use { topograph::{ executor::{Executor, Nonblock, Tokio}, prelude::*, + AsyncHandler, }, tracing::{debug, warn}, yellowstone_grpc_client::GeyserGrpcClient, @@ -34,6 +35,89 @@ enum GrpcJob { ProcessSubscribeUpdate(Box), } +#[derive(Clone)] +pub struct GrpcJobHandler { + connection: redis::aio::MultiplexedConnection, + config: Arc, + pipe: Arc>, +} + +impl<'a> AsyncHandler>> + for GrpcJobHandler +{ + type Output = (); + + fn handle( + &self, + job: GrpcJob, + handle: topograph::executor::Handle<'a, GrpcJob, Nonblock>, + ) -> impl futures::Future + Send + 'a { + let config = Arc::clone(&self.config); + let connection = self.connection.clone(); + let pipe = Arc::clone(&self.pipe); + + async move { + match job { + GrpcJob::FlushRedisPipe => { + let mut pipe = pipe.lock().await; + let mut connection = connection; + + let flush = pipe.flush(&mut connection).await; + + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let counts = flush.as_ref().unwrap_or_else(|counts| counts); + + for (stream, count) in counts.iter() { + redis_xadd_status_inc(stream, status, *count); + } + + debug!(message = "Redis pipe flushed", ?status, ?counts); + } + GrpcJob::ProcessSubscribeUpdate(update) => { + let accounts_stream = config.accounts.stream.clone(); + let accounts_stream_maxlen = config.accounts.stream_maxlen; + let transactions_stream = config.transactions.stream.clone(); + let transactions_stream_maxlen = config.transactions.stream_maxlen; + + let SubscribeUpdate { update_oneof, .. } = *update; + + let mut pipe = pipe.lock().await; + + if let Some(update) = update_oneof { + match update { + UpdateOneof::Account(account) => { + pipe.xadd_maxlen( + &accounts_stream, + StreamMaxlen::Approx(accounts_stream_maxlen), + "*", + account.encode_to_vec(), + ); + + debug!(message = "Account update", ?account,); + } + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &transactions_stream, + StreamMaxlen::Approx(transactions_stream_maxlen), + "*", + transaction.encode_to_vec(), + ); + + debug!(message = "Transaction update", ?transaction); + } + var => warn!(message = "Unknown update variant", ?var), + } + } + + if pipe.size() >= config.redis.pipeline_max_size { + handle.push(GrpcJob::FlushRedisPipe); + } + } + } + } + } +} + pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { let redis_client = redis::Client::open(config.redis.url.clone())?; let config = Arc::new(config); @@ -72,76 +156,12 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { .await?; tokio::pin!(stream); - let pool_config = Arc::clone(&config); - let pool_connection = connection.clone(); - let pool_pipe = Arc::clone(&pipe); - let exec = Executor::builder(Nonblock(Tokio)) - .num_threads(Some(config.topograph.num_threads)) - .build(move |update, _handle| { - let config = Arc::clone(&pool_config); - let connection = pool_connection.clone(); - let pipe = Arc::clone(&pool_pipe); - - async move { - match update { - GrpcJob::FlushRedisPipe => { - let mut pipe = pipe.lock().await; - let mut connection = connection; - - let flush = pipe.flush(&mut connection).await; - - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); - - for (stream, count) in counts.iter() { - redis_xadd_status_inc(stream, status, *count); - } - - debug!(message = "Redis pipe flushed", ?status, ?counts); - } - GrpcJob::ProcessSubscribeUpdate(update) => { - let accounts_stream = config.accounts.stream.clone(); - let accounts_stream_maxlen = config.accounts.stream_maxlen; - let transactions_stream = config.transactions.stream.clone(); - let transactions_stream_maxlen = config.transactions.stream_maxlen; - - let SubscribeUpdate { update_oneof, .. } = *update; - - let mut pipe = pipe.lock().await; - - if let Some(update) = update_oneof { - match update { - UpdateOneof::Account(account) => { - pipe.xadd_maxlen( - &accounts_stream, - StreamMaxlen::Approx(accounts_stream_maxlen), - "*", - account.encode_to_vec(), - ); - - debug!(message = "Account update", ?account,); - } - UpdateOneof::Transaction(transaction) => { - pipe.xadd_maxlen( - &transactions_stream, - StreamMaxlen::Approx(transactions_stream_maxlen), - "*", - transaction.encode_to_vec(), - ); - - debug!(message = "Transaction update", ?transaction); - } - var => warn!(message = "Unknown update variant", ?var), - } - } - - if pipe.size() >= config.redis.pipeline_max_size { - // handle.push(GrpcJob::FlushRedisPipe); - } - } - } - } + .max_concurrency(Some(config.max_concurrency)) + .build_async(GrpcJobHandler { + config: Arc::clone(&config), + connection: connection.clone(), + pipe: Arc::clone(&pipe), })?; let deadline_config = Arc::clone(&config); diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 17ce8c414..f2a00994d 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,16 +1,14 @@ use { crate::{ - config::{ConfigIngester, ConfigIngesterDownloadMetadata}, + config::{ConfigIngester, ConfigIngesterDownloadMetadata, REDIS_STREAM_DATA_KEY}, download_metadata::TASK_TYPE, postgres::{create_pool as pg_create_pool, metrics_pgpool, report_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, - program_transformer_tasks_total_set, redis_xadd_status_inc, - ProgramTransformerTaskStatusKind, + program_transformer_tasks_total_set, ProgramTransformerTaskStatusKind, }, redis::{ - metrics_xlen, report_xlen, ProgramTransformerInfo, RedisStream, RedisStreamMessageInfo, - TrackedPipeline, + metrics_xlen, IngestStream, ProgramTransformerInfo, RedisStream, RedisStreamMessage, }, util::create_shutdown, }, @@ -22,8 +20,10 @@ use { future::{pending, BoxFuture, FusedFuture, FutureExt}, stream::StreamExt, }, - program_transformers::{error::ProgramTransformerError, ProgramTransformer}, - redis::{aio::MultiplexedConnection, streams::StreamMaxlen}, + program_transformers::{ + error::ProgramTransformerError, AccountInfo, ProgramTransformer, TransactionInfo, + }, + redis::aio::MultiplexedConnection, sea_orm::{ entity::{ActiveModelTrait, ActiveValue}, error::{DbErr, RuntimeErr}, @@ -38,58 +38,46 @@ use { }, }, tokio::{ - sync::Mutex, task::JoinSet, time::{sleep, Duration}, }, - topograph::{ - executor::{Executor, Nonblock, Tokio}, - prelude::*, - }, - tracing::{debug, error, warn}, + tracing::warn, }; -enum IngestJob { - SaveMessage(RedisStreamMessageInfo), - FlushRedisPipe(Arc>, MultiplexedConnection), -} - -enum ReportJob { - Redis(Vec, MultiplexedConnection), - Postgres(PgPool), -} - fn download_metadata_notifier_v2( - pipe: Arc>, + connection: MultiplexedConnection, stream: String, stream_maxlen: usize, ) -> anyhow::Result { Ok( - Box::new( - move |info: DownloadMetadataInfo| -> BoxFuture< - 'static, - Result<(), Box>, - > { - let pipe = Arc::clone(&pipe); - let stream = stream.clone(); - Box::pin(async move { - download_metadata_inserted_total_inc(); - - let mut pipe = pipe.lock().await; - let info_bytes = serde_json::to_vec(&info)?; - - pipe.xadd_maxlen( - &stream, - StreamMaxlen::Approx(stream_maxlen), - "*", - info_bytes, - ); - - Ok(()) - }) - }, - ), - ) + Box::new( + move |info: DownloadMetadataInfo| -> BoxFuture< + 'static, + Result<(), Box>, + > { + let mut connection = connection.clone(); + let stream = stream.clone(); + Box::pin(async move { + download_metadata_inserted_total_inc(); + + let info_bytes = serde_json::to_vec(&info)?; + + redis::cmd("XADD") + .arg(&stream) + .arg("MAXLEN") + .arg("~") + .arg(stream_maxlen) + .arg("*") + .arg(REDIS_STREAM_DATA_KEY) + .arg(info_bytes) + .query_async(&mut connection) + .await?; + + Ok(()) + }) + }, + ), + ) } pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { @@ -97,28 +85,16 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { let connection = redis_client.get_multiplexed_tokio_connection().await?; let pool = pg_create_pool(config.postgres).await?; - let pipe = Arc::new(Mutex::new(TrackedPipeline::default())); - let (mut redis_messages, redis_tasks_fut) = - RedisStream::new(config.redis.clone(), connection.clone()).await?; - tokio::pin!(redis_tasks_fut); - - let streams = config - .redis - .streams - .iter() - .map(|config| config.stream.to_string()) - .collect::>(); - let download_metadata_stream = config.download_metadata.stream.clone(); let download_metadata_stream_maxlen = config.download_metadata.stream_maxlen; let accounts_download_metadata_notifier = download_metadata_notifier_v2( - Arc::clone(&pipe), + connection.clone(), download_metadata_stream.clone(), download_metadata_stream_maxlen, )?; let transactions_download_metadata_notifier = download_metadata_notifier_v2( - Arc::clone(&pipe), + connection.clone(), download_metadata_stream.clone(), download_metadata_stream_maxlen, )?; @@ -136,148 +112,75 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { .build()?; let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); - let exec = Executor::builder(Nonblock(Tokio)) - .num_threads(Some(config.topograph.num_threads)) - .build(move |update, _handle| { - let pt_accounts = Arc::clone(&pt_accounts); - let pt_transactions = Arc::clone(&pt_transactions); + let download_metadata_stream = IngestStream::build() + .config(config.download_metadata.stream_config.clone()) + .connection(connection.clone()) + .handler(move |info| { let download_metadata = Arc::clone(&download_metadata); - async move { - match update { - IngestJob::FlushRedisPipe(pipe, connection) => { - let mut pipe = pipe.lock().await; - let mut connection = connection; - - let flush = pipe.flush(&mut connection).await; - - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); + Box::pin(async move { + let info = DownloadMetadataInfo::try_parse_msg(info)?; - for (stream, count) in counts.iter() { - redis_xadd_status_inc(stream, status, *count); - } - - debug!(message = "Redis pipe flushed", ?status, ?counts); - } - IngestJob::SaveMessage(msg) => { - let result = match &msg.get_data() { - ProgramTransformerInfo::Account(account) => { - pt_accounts.handle_account_update(account).await - } - ProgramTransformerInfo::Transaction(transaction) => { - pt_transactions.handle_transaction(transaction).await - } - ProgramTransformerInfo::MetadataJson(download_metadata_info) => { - download_metadata - .handle_download(download_metadata_info) - .await - .map_err(|e| { - ProgramTransformerError::AssetIndexError(e.to_string()) - }) - } - }; - match result { - Ok(()) => program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::Success, - ), - Err(ProgramTransformerError::NotImplemented) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::NotImplemented, - ); - error!("not implemented") - } - Err(ProgramTransformerError::DeserializationError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::DeserializationError, - ); + download_metadata + .handle_download(&info) + .await + .map_err(Into::into) + }) + }) + .start()?; + let account_stream = IngestStream::build() + .config(config.accounts.clone()) + .connection(connection.clone()) + .handler(move |info| { + let pt_accounts = Arc::clone(&pt_accounts); - error!("failed to deserialize {:?}", error) - } - Err(ProgramTransformerError::ParsingError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::ParsingError, - ); + Box::pin(async move { + let info = AccountInfo::try_parse_msg(info)?; - error!("failed to parse {:?}", error) - } - Err(ProgramTransformerError::DatabaseError(error)) => { - error!("database error for {:?}", error) - } - Err(ProgramTransformerError::AssetIndexError(error)) => { - error!("indexing error for {:?}", error) - } - Err(error) => { - error!("failed to handle {:?}", error) - } - } + pt_accounts + .handle_account_update(&info) + .await + .map_err(Into::into) + }) + }) + .start()?; + let transactions_stream = IngestStream::build() + .config(config.transactions.clone()) + .connection(connection.clone()) + .handler(move |info| { + let pt_transactions = Arc::clone(&pt_transactions); + Box::pin(async move { + let info = TransactionInfo::try_parse_msg(info)?; + + pt_transactions + .handle_transaction(&info) + .await + .map_err(Into::into) + }) + }) + .start()?; - if let Err(e) = msg.ack() { - error!("Failed to acknowledge message: {:?}", e); - } else { - debug!("Message acknowledged successfully"); - } - } - } - } - })?; + let mut shutdown = create_shutdown()?; - let reporting_exec = Executor::builder(Nonblock(Tokio)) - .num_threads(Some(1)) - .build(move |update, _handle| async move { - match update { - ReportJob::Postgres(pool) => { - report_pgpool(pool); + let reporting_pool = pool.clone(); - debug!("Successfully reported Postgres pool metrics"); - } - ReportJob::Redis(streams, connection) => { - match report_xlen(connection, streams).await { - Ok(_) => debug!("Successfully reported Redis stream length"), - Err(e) => error!("Failed to report Redis stream length: {:?}", e), - } - } - } - })?; + tokio::spawn(async move { + loop { + report_pgpool(reporting_pool.clone()); - let mut shutdown = create_shutdown()?; + sleep(Duration::from_millis(100)).await; + } + }); - loop { - tokio::select! { - _ = sleep(Duration::from_millis(100)) => { - reporting_exec.push(ReportJob::Postgres(pool.clone())); - } - _ = sleep(Duration::from_millis(100)) => { - reporting_exec.push(ReportJob::Redis(streams.clone(), connection.clone())); - } - _ = sleep(config.download_metadata.pipeline_max_idle) => { - exec.push(IngestJob::FlushRedisPipe(Arc::clone(&pipe), connection.clone())); - } - Some(msg) = redis_messages.recv() => { - exec.push(IngestJob::SaveMessage(msg)); - } - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - break; - } - result = &mut redis_tasks_fut => { - if let Err(error) = result { - error!("Error in redis_tasks_fut: {:?}", error); - } - break; - } + tokio::select! { + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); } } - exec.push(IngestJob::FlushRedisPipe( - Arc::clone(&pipe), - connection.clone(), - )); - - redis_messages.shutdown(); - - exec.join_async().await; - reporting_exec.join_async().await; + account_stream.stop().await?; + transactions_stream.stop().await?; + download_metadata_stream.stop().await?; pool.close().await; diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index e81de4255..6beab4614 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -1,10 +1,12 @@ use { - crate::version::VERSION as VERSION_INFO, + crate::{redis::RedisStreamMessageError, version::VERSION as VERSION_INFO}, + das_core::MetadataJsonTaskError, hyper::{ server::conn::AddrStream, service::{make_service_fn, service_fn}, Body, Request, Response, Server, StatusCode, }, + program_transformers::error::ProgramTransformerError, prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder}, std::{net::SocketAddr, sync::Once}, tracing::{error, info}, @@ -164,16 +166,107 @@ pub enum ProgramTransformerTaskStatusKind { NotImplemented, DeserializationError, ParsingError, + ChangeLogEventMalformed, + StorageWriteError, + SerializatonError, + DatabaseError, + AssetIndexError, + DownloadMetadataNotify, + DownloadMetadataSeaOrmError, + DownloadMetadataFetchError, + DownloadMetadataAssetNotFound, + RedisMessageDeserializeError, } -pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKind) { - PROGRAM_TRANSFORMER_TASK_STATUS - .with_label_values(&[match kind { +impl From for ProgramTransformerTaskStatusKind { + fn from(error: ProgramTransformerError) -> Self { + match error { + ProgramTransformerError::ChangeLogEventMalformed => { + ProgramTransformerTaskStatusKind::ChangeLogEventMalformed + } + ProgramTransformerError::StorageWriteError(_) => { + ProgramTransformerTaskStatusKind::StorageWriteError + } + ProgramTransformerError::NotImplemented => { + ProgramTransformerTaskStatusKind::NotImplemented + } + ProgramTransformerError::DeserializationError(_) => { + ProgramTransformerTaskStatusKind::DeserializationError + } + ProgramTransformerError::SerializatonError(_) => { + ProgramTransformerTaskStatusKind::SerializatonError + } + ProgramTransformerError::ParsingError(_) => { + ProgramTransformerTaskStatusKind::ParsingError + } + ProgramTransformerError::DatabaseError(_) => { + ProgramTransformerTaskStatusKind::DatabaseError + } + ProgramTransformerError::AssetIndexError(_) => { + ProgramTransformerTaskStatusKind::AssetIndexError + } + ProgramTransformerError::DownloadMetadataNotify(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataNotify + } + } + } +} + +impl From for ProgramTransformerTaskStatusKind { + fn from(error: MetadataJsonTaskError) -> Self { + match error { + MetadataJsonTaskError::SeaOrm(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataSeaOrmError + } + MetadataJsonTaskError::Fetch(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataFetchError + } + MetadataJsonTaskError::AssetNotFound => { + ProgramTransformerTaskStatusKind::DownloadMetadataAssetNotFound + } + } + } +} + +impl From for ProgramTransformerTaskStatusKind { + fn from(_: RedisStreamMessageError) -> Self { + ProgramTransformerTaskStatusKind::RedisMessageDeserializeError + } +} +impl ProgramTransformerTaskStatusKind { + pub const fn to_str(self) -> &'static str { + match self { ProgramTransformerTaskStatusKind::Success => "success", ProgramTransformerTaskStatusKind::NotImplemented => "not_implemented", ProgramTransformerTaskStatusKind::DeserializationError => "deserialization_error", ProgramTransformerTaskStatusKind::ParsingError => "parsing_error", - }]) + ProgramTransformerTaskStatusKind::ChangeLogEventMalformed => { + "changelog_event_malformed" + } + ProgramTransformerTaskStatusKind::StorageWriteError => "storage_write_error", + ProgramTransformerTaskStatusKind::SerializatonError => "serialization_error", + ProgramTransformerTaskStatusKind::DatabaseError => "database_error", + ProgramTransformerTaskStatusKind::AssetIndexError => "asset_index_error", + ProgramTransformerTaskStatusKind::DownloadMetadataNotify => "download_metadata_notify", + ProgramTransformerTaskStatusKind::DownloadMetadataSeaOrmError => { + "download_metadata_sea_orm_error" + } + ProgramTransformerTaskStatusKind::DownloadMetadataFetchError => { + "download_metadata_fetch_error" + } + ProgramTransformerTaskStatusKind::DownloadMetadataAssetNotFound => { + "download_metadata_asset_not_found" + } + ProgramTransformerTaskStatusKind::RedisMessageDeserializeError => { + "redis_message_deserialize_error" + } + } + } +} + +pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKind) { + PROGRAM_TRANSFORMER_TASK_STATUS + .with_label_values(&[kind.to_str()]) .inc() } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 8e0d5d5a9..fc7bcd430 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -1,7 +1,13 @@ use { crate::{ - config::{ConfigIngesterRedis, ConfigIngesterRedisStreamType, REDIS_STREAM_DATA_KEY}, - prom::{redis_xack_inc, redis_xlen_set}, + config::{ + ConfigIngestStream, ConfigIngesterRedis, ConfigIngesterRedisStreamType, + REDIS_STREAM_DATA_KEY, + }, + prom::{ + program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, + ProgramTransformerTaskStatusKind, + }, }, das_core::DownloadMetadataInfo, futures::future::{BoxFuture, Fuse, FutureExt}, @@ -9,8 +15,8 @@ use { redis::{ aio::MultiplexedConnection, streams::{ - StreamClaimReply, StreamId, StreamKey, StreamMaxlen, StreamPendingCountReply, - StreamReadOptions, StreamReadReply, + StreamClaimOptions, StreamClaimReply, StreamId, StreamKey, StreamMaxlen, + StreamPendingCountReply, StreamReadOptions, StreamReadReply, }, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, @@ -28,6 +34,12 @@ use { task::JoinSet, time::{sleep, Duration, Instant}, }, + topograph::{ + executor::{Executor, Nonblock, Tokio}, + prelude::*, + AsyncHandler, + }, + tracing::{debug, error, info}, yellowstone_grpc_proto::{ convert_from::{ create_message_instructions, create_meta_inner_instructions, create_pubkey_vec, @@ -37,6 +49,497 @@ use { }, }; +pub enum IngestStreamJob { + Process((String, HashMap)), +} + +pub struct IngestStreamStop { + shutdown_tx: tokio::sync::oneshot::Sender<()>, + control: tokio::task::JoinHandle<()>, +} + +impl IngestStreamStop { + pub async fn stop(self) -> anyhow::Result<()> { + let _ = self.shutdown_tx.send(()); + + self.control.await?; + // self.executor.join_async().await; + + Ok(()) + } +} + +type HandlerFn = dyn Fn(HashMap) -> BoxFuture<'static, Result<(), IngestMessageError>> + + Send + + Sync; + +#[derive(Clone)] +pub struct IngestStreamHandler { + handler: Arc, + ack_sender: tokio::sync::mpsc::Sender, +} + +impl<'a> + AsyncHandler>> + for IngestStreamHandler +{ + type Output = (); + + fn handle( + &self, + job: IngestStreamJob, + _handle: topograph::executor::Handle<'a, IngestStreamJob, Nonblock>, + ) -> impl futures::Future + Send { + let handler = Arc::clone(&self.handler); + let ack_sender = self.ack_sender.clone(); + + async move { + match job { + IngestStreamJob::Process((id, msg)) => { + match handler(msg).await { + Ok(()) => program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::Success, + ), + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + } + + if let Err(e) = ack_sender.send(id).await { + error!("Failed to send ack id to channel: {:?}", e); + } + } + } + } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum RedisStreamMessageError { + #[error("failed to get data (key: {0}) from stream")] + MissingData(String), + #[error("invalid data (key: {0}) from stream")] + InvalidData(String), + #[error("failed to decode message")] + Decode(#[from] yellowstone_grpc_proto::prost::DecodeError), + #[error("received invalid SubscribeUpdateAccount")] + InvalidSubscribeUpdateAccount, + #[error("failed to convert pubkey")] + PubkeyConversion(#[from] std::array::TryFromSliceError), + #[error("JSON deserialization error: {0}")] + JsonDeserialization(#[from] serde_json::Error), +} + +pub trait RedisStreamMessage { + fn try_parse_msg(msg: HashMap) -> Result; + + fn get_data_as_vec( + msg: &HashMap, + ) -> Result<&Vec, RedisStreamMessageError> { + let data = msg.get(REDIS_STREAM_DATA_KEY).ok_or_else(|| { + RedisStreamMessageError::MissingData(REDIS_STREAM_DATA_KEY.to_string()) + })?; + + match data { + RedisValue::Data(data) => Ok(data), + _ => Err(RedisStreamMessageError::InvalidData( + REDIS_STREAM_DATA_KEY.to_string(), + )), + } + } +} + +impl RedisStreamMessage for AccountInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let account_data = Self::get_data_as_vec(&msg)?; + + let SubscribeUpdateAccount { account, slot, .. } = Message::decode(account_data.as_ref())?; + + let account = + account.ok_or_else(|| RedisStreamMessageError::InvalidSubscribeUpdateAccount)?; + + Ok(Self { + slot, + pubkey: Pubkey::try_from(account.pubkey.as_slice())?, + owner: Pubkey::try_from(account.owner.as_slice())?, + data: account.data, + }) + } +} + +impl RedisStreamMessage for TransactionInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let transaction_data = Self::get_data_as_vec(&msg)?; + + let SubscribeUpdateTransaction { transaction, slot } = + Message::decode(transaction_data.as_ref())?; + + let transaction = transaction.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid SubscribeUpdateTransaction".to_string(), + ) + })?; + let tx = transaction.transaction.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid transaction in SubscribeUpdateTransaction".to_string(), + ) + })?; + let message = tx.message.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid message in SubscribeUpdateTransaction".to_string(), + ) + })?; + let meta = transaction.meta.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid meta in SubscribeUpdateTransaction".to_string(), + ) + })?; + + let mut account_keys = create_pubkey_vec(message.account_keys).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })?; + for pubkey in create_pubkey_vec(meta.loaded_writable_addresses).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })? { + account_keys.push(pubkey); + } + for pubkey in create_pubkey_vec(meta.loaded_readonly_addresses).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })? { + account_keys.push(pubkey); + } + + Ok(Self { + slot, + signature: Signature::try_from(transaction.signature.as_slice())?, + account_keys, + message_instructions: create_message_instructions(message.instructions).map_err( + |e| { + RedisStreamMessageError::Decode( + yellowstone_grpc_proto::prost::DecodeError::new(e), + ) + }, + )?, + meta_inner_instructions: create_meta_inner_instructions(meta.inner_instructions) + .map_err(|e| { + RedisStreamMessageError::Decode( + yellowstone_grpc_proto::prost::DecodeError::new(e), + ) + })?, + }) + } +} + +impl RedisStreamMessage for DownloadMetadataInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let metadata_data = Self::get_data_as_vec(&msg)?; + + let info: DownloadMetadataInfo = serde_json::from_slice(metadata_data.as_ref())?; + + Ok(info) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum IngestMessageError { + #[error("Redis stream message parse error: {0}")] + RedisStreamMessage(#[from] RedisStreamMessageError), + #[error("Program transformer error: {0}")] + ProgramTransformer(#[from] program_transformers::error::ProgramTransformerError), + #[error("Download metadata JSON task error: {0}")] + DownloadMetadataJson(#[from] das_core::MetadataJsonTaskError), +} + +pub struct IngestStream { + config: ConfigIngestStream, + connection: Option, + handler: Option>, +} + +impl IngestStream { + pub fn build() -> Self { + Self { + config: ConfigIngestStream::default(), + connection: None, + handler: None, + } + } + + pub fn config(mut self, config: ConfigIngestStream) -> Self { + self.config = config; + self + } + + pub fn connection(mut self, connection: MultiplexedConnection) -> Self { + self.connection = Some(connection); + self + } + + pub fn handler(mut self, handler: F) -> Self + where + F: Fn(HashMap) -> BoxFuture<'static, Result<(), IngestMessageError>> + + Send + + Sync + + 'static, + { + self.handler = Some(Arc::new(handler)); + self + } + + pub fn start(mut self) -> anyhow::Result { + let config = Arc::new(self.config); + + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + + let connection = self.connection.take().expect("Connection is required"); + + let group_create_connection = connection.clone(); + let config_group_create = Arc::clone(&config); + + tokio::task::spawn_blocking(move || { + let mut connection = group_create_connection.clone(); + let config = Arc::clone(&config_group_create); + + let rt = tokio::runtime::Runtime::new()?; + + rt.block_on(async { + if let Err(e) = xgroup_create( + &mut connection, + &config.name, + &config.group, + &config.consumer, + ) + .await + { + error!("Failed to create group: {:?}", e); + } else { + debug!( + "Group created successfully: name={}, group={}, consumer={}", + config.name, config.group, config.consumer + ); + } + }); + + Ok::<(), anyhow::Error>(()) + }); + + let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); + let (ack_shutdown_tx, ack_shutdown_rx) = tokio::sync::oneshot::channel(); + + tokio::spawn({ + let config = Arc::clone(&config); + let mut connection = connection.clone(); + let mut pending_ids = Vec::new(); + + async move { + let mut shutdown_rx = ack_shutdown_rx; + let deadline = tokio::time::sleep(config.xack_batch_max_idle); + tokio::pin!(deadline); + + loop { + tokio::select! { + Some(id) = ack_rx.recv() => { + pending_ids.push(id); + if pending_ids.len() >= config.xack_batch_max_size { + let ids = std::mem::take(&mut pending_ids); + match redis::pipe() + .atomic() + .xack(&config.name, &config.group, &ids) + .xdel(&config.name, &ids) + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(_) => { + redis_xack_inc(&config.name, ids.len()); + } + Err(e) => { + error!("Failed to acknowledge messages: {:?}", e); + } + } + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); + } + }, + _ = &mut deadline => { + if !pending_ids.is_empty() { + let ids = std::mem::take(&mut pending_ids); + match redis::pipe() + .atomic() + .xack(&config.name, &config.group, &ids) + .xdel(&config.name, &ids) + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(_) => { + redis_xack_inc(&config.name, ids.len()); + } + Err(e) => { + error!("Failed to acknowledge messages: {:?}", e); + } + } + } + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); + }, + _ = &mut shutdown_rx => { + break; + } + } + } + } + }); + + let handler = self.handler.take().expect("Handler is required"); + + let executor = Executor::builder(Nonblock(Tokio)) + .max_concurrency(Some(config.max_concurrency)) + .build_async(IngestStreamHandler { + handler, + ack_sender: ack_tx.clone(), + })?; + + let connection_report = connection.clone(); + let streams_report = vec![config.name.clone()]; + + tokio::spawn(async move { + loop { + let connection = connection_report.clone(); + let streams = streams_report.clone(); + + if let Err(e) = report_xlen(connection, streams).await { + error!("Failed to report xlen: {:?}", e); + } + + sleep(Duration::from_millis(100)).await; + } + }); + + let config_read = Arc::clone(&config); + let mut connection_read = connection.clone(); + + let (read_shutdown_tx, read_shutdown_rx) = tokio::sync::oneshot::channel(); + + tokio::spawn(async move { + debug!("Starting read stream task name={}", config_read.name); + + let mut shutdown_rx = read_shutdown_rx; + + loop { + if shutdown_rx.try_recv().is_ok() { + debug!( + "Shutdown signal received, exiting prefetch loop name={}", + config_read.name + ); + break; + } + + if let Ok(pending) = redis::cmd("XPENDING") + .arg(&config_read.name) + .arg(&config_read.group) + .arg("-") + .arg("+") + .arg(config_read.batch_size) + .arg(&config_read.consumer) + .query_async::<_, StreamPendingCountReply>(&mut connection_read) + .await + { + if pending.ids.is_empty() { + debug!( + "No pending messages stream={} consumer={} group={}", + config_read.name, config_read.consumer, config_read.group + ); + break; + } + + let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); + let claim_opts = StreamClaimOptions::default(); + + let claimed: RedisResult = connection_read + .xclaim_options( + &config_read.name, + &config_read.group, + &config_read.consumer, + 20, + &ids, + claim_opts, + ) + .await; + + if let Ok(claimed) = claimed { + for StreamId { id, map } in claimed.ids { + executor.push(IngestStreamJob::Process((id, map))); + } + } + } + } + + loop { + if shutdown_rx.try_recv().is_ok() { + debug!( + "Shutdown signal received, exiting read loop name={}", + config_read.name + ); + break; + } + + let opts = StreamReadOptions::default() + .group(&config_read.group, &config_read.consumer) + .count(config_read.batch_size) + .block(20); + + let result: RedisResult = connection_read + .xread_options(&[&config_read.name], &[">"], &opts) + .await; + info!( + "Read from stream name={} result={:?}", + config_read.name, result + ); + + match result { + Ok(reply) => { + for StreamKey { key: _, ids } in reply.keys { + for StreamId { id, map } in ids { + debug!("Reading and processing: id={} msg={:?}", id, map); + executor.push(IngestStreamJob::Process((id, map))); + } + } + } + Err(err) => { + error!("Error reading from stream: {:?}", err); + } + } + } + }); + + let control = tokio::spawn(async move { + let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = shutdown_rx; + debug!("Starting ingest stream name={}", config.name); + + tokio::select! { + _ = &mut shutdown_rx => { + info!("Shut down ingest stream name={}", config.name); + + let _ = read_shutdown_tx.send(()); + let _ = ack_shutdown_tx.send(()); + } + } + }); + + Ok(IngestStreamStop { + control, + shutdown_tx, + }) + } +} + pub async fn report_xlen( mut connection: C, streams: Vec, @@ -231,6 +734,70 @@ pub struct RedisStream { messages_rx: mpsc::Receiver, } +#[allow(dead_code)] +async fn run_ack( + stream: Arc, + connection: MultiplexedConnection, + mut ack_rx: mpsc::UnboundedReceiver, +) -> anyhow::Result<()> { + let mut ids = vec![]; + let deadline = sleep(stream.xack_batch_max_idle); + tokio::pin!(deadline); + let mut tasks = JoinSet::new(); + + let result = loop { + let terminated = tokio::select! { + msg = ack_rx.recv() => match msg { + Some(msg) => { + ids.push(msg); + if ids.len() < stream.xack_batch_max_size { + continue; + } + false + } + None => true, + }, + _ = &mut deadline => false, + }; + + let ids = std::mem::take(&mut ids); + deadline + .as_mut() + .reset(Instant::now() + stream.xack_batch_max_idle); + if !ids.is_empty() { + tasks.spawn({ + let stream = Arc::clone(&stream); + let mut connection = connection.clone(); + async move { + redis::pipe() + .atomic() + .xack(&stream.stream_name, &stream.group, &ids) + .xdel(&stream.stream_name, &ids) + .query_async(&mut connection) + .await?; + redis_xack_inc(&stream.stream_name, ids.len()); + Ok::<(), anyhow::Error>(()) + } + }); + while tasks.len() >= stream.xack_max_in_process { + if let Some(result) = tasks.join_next().await { + result??; + } + } + } + + if terminated { + break Ok(()); + } + }; + + while let Some(result) = tasks.join_next().await { + result??; + } + + result +} + impl RedisStream { pub async fn new( config: ConfigIngesterRedis, @@ -380,6 +947,7 @@ impl RedisStream { let streams_keys = streams.keys().collect::>(); let streams_ids = (0..streams_keys.len()).map(|_| ">").collect::>(); + while !shutdown.load(Ordering::Relaxed) { let opts = StreamReadOptions::default() .count(config.xreadgroup_max) @@ -387,6 +955,7 @@ impl RedisStream { let results: StreamReadReply = connection .xread_options(&streams_keys, &streams_ids, &opts) .await?; + if results.keys.is_empty() { sleep(Duration::from_millis(5)).await; continue; diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index aada07fc0..de4eaca6d 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -20,6 +20,7 @@ use { entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, }, + serde::Deserialize, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, sqlx::PgPool, @@ -35,7 +36,7 @@ mod mpl_core_program; mod token; mod token_metadata; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct AccountInfo { pub slot: u64, pub pubkey: Pubkey, @@ -43,7 +44,7 @@ pub struct AccountInfo { pub data: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct TransactionInfo { pub slot: u64, pub signature: Signature, From 0db061275c70b8eaf83ab687cda75455841c4b70 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 22 Aug 2024 10:27:13 +0200 Subject: [PATCH 33/72] fix: config and ack --- grpc-ingest/config-ingester.yml | 17 +- grpc-ingest/src/config.rs | 22 ++- grpc-ingest/src/ingester.rs | 4 +- grpc-ingest/src/redis.rs | 148 ++++++++++++------ .../src/token_metadata/v1_asset.rs | 20 +-- 5 files changed, 139 insertions(+), 72 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 56cd50b10..de4faa653 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -3,7 +3,17 @@ redis: url: redis://localhost:6379 group: ingester consumer: consumer # every new ingester instance should have uniq name - streams: [] + + streams: + - type: account + stream: ACCOUNTS + - type: transaction + stream: TRANSACTIONS + xack_batch_max_size: 100 + xack_batch_max_idle_ms: 10 + xack_max_in_process: 100 + - type: metadatajson + stream: METADATA_JSON prefetch_queue_size: 5_000 # max number of messages available in the read queue for processing xpending_max: 1_000 # used for reading pending messages xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) @@ -16,6 +26,11 @@ program_transformer: max_tasks_in_process: 100 accounts: name: ACCOUNTS + max_concurrency: 10 + ack_buffer_size: 10 + batch_size: 100 + xack_batch_max_idle_ms: 1_000 + xack_buffer_size: 10_000 transactions: name: TRANSACTIONS download_metadata: diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index a27781e9d..f12b2cdef 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -39,7 +39,10 @@ pub struct ConfigIngestStream { pub group: String, #[serde(default = "ConfigIngestStream::default_consumer")] pub consumer: String, - #[serde(default = "ConfigIngestStream::default_xack_batch_max_size")] + #[serde( + default = "ConfigIngestStream::default_xack_batch_max_size", + deserialize_with = "deserialize_usize_str" + )] pub xack_batch_max_size: usize, #[serde( default = "ConfigIngestStream::default_xack_batch_max_idle", @@ -47,7 +50,10 @@ pub struct ConfigIngestStream { rename = "xack_batch_max_idle_ms" )] pub xack_batch_max_idle: Duration, - #[serde(default = "ConfigIngestStream::default_batch_size")] + #[serde( + default = "ConfigIngestStream::default_batch_size", + deserialize_with = "deserialize_usize_str" + )] pub batch_size: usize, #[serde( default = "ConfigIngestStream::default_max_concurrency", @@ -70,7 +76,7 @@ impl ConfigIngestStream { } pub const fn default_xack_batch_max_idle() -> Duration { - Duration::from_millis(100) + Duration::from_millis(10_000) } pub fn default_group() -> String { @@ -82,11 +88,11 @@ impl ConfigIngestStream { } pub const fn default_xack_batch_max_size() -> usize { - 100 + 5 } pub const fn default_batch_size() -> usize { - 100 + 5 } } @@ -256,9 +262,9 @@ pub struct ConfigIngester { impl ConfigIngester { pub fn check(&self) { - let total_threads = self.program_transformer.account_num_threads - + self.program_transformer.transaction_num_threads - + self.program_transformer.metadata_json_num_threads; + let total_threads = self.accounts.max_concurrency + + self.transactions.max_concurrency + + self.download_metadata.stream_config.max_concurrency; if self.postgres.max_connections < total_threads { warn!( diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index f2a00994d..54391efc9 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -291,9 +291,7 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { ProgramTransformerInfo::Transaction(transaction) => { pt_transactions.handle_transaction(transaction).await } - ProgramTransformerInfo::MetadataJson(_download_metadata_info) => { - todo!() - } + ProgramTransformerInfo::MetadataJson(_download_metadata_info) => Ok(()), }; macro_rules! log_or_bail { diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index fc7bcd430..26c3d0014 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -260,6 +260,56 @@ pub enum IngestMessageError { DownloadMetadataJson(#[from] das_core::MetadataJsonTaskError), } +#[derive(Clone, Debug)] +enum AcknowledgeJob { + Submit(Vec), +} + +#[derive(Clone)] +pub struct AcknowledgeHandler { + config: Arc, + connection: MultiplexedConnection, +} + +impl<'a> + AsyncHandler>> + for AcknowledgeHandler +{ + type Output = (); + + fn handle( + &self, + job: AcknowledgeJob, + _handle: topograph::executor::Handle<'a, AcknowledgeJob, Nonblock>, + ) -> impl futures::Future + Send { + let mut connection = self.connection.clone(); + let config = Arc::clone(&self.config); + + let AcknowledgeJob::Submit(ids) = job; + + let count = ids.len(); + + async move { + match redis::pipe() + .atomic() + .xack(&config.name, &config.group, &ids) + .xdel(&config.name, &ids) + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(response) => { + info!("Acknowledged and deleted message: response={:?}", response); + + redis_xack_inc(&config.name, count); + } + Err(e) => { + error!("Failed to acknowledge or delete message: error={:?}", e); + } + } + } + } +} + pub struct IngestStream { config: ConfigIngestStream, connection: Option, @@ -336,10 +386,16 @@ impl IngestStream { let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); let (ack_shutdown_tx, ack_shutdown_rx) = tokio::sync::oneshot::channel(); + let xack_executor = Executor::builder(Nonblock(Tokio)) + .max_concurrency(Some(10)) + .build_async(AcknowledgeHandler { + config: Arc::clone(&config), + connection: connection.clone(), + })?; + tokio::spawn({ let config = Arc::clone(&config); - let mut connection = connection.clone(); - let mut pending_ids = Vec::new(); + let mut pending = Vec::new(); async move { let mut shutdown_rx = ack_shutdown_rx; @@ -349,47 +405,24 @@ impl IngestStream { loop { tokio::select! { Some(id) = ack_rx.recv() => { - pending_ids.push(id); - if pending_ids.len() >= config.xack_batch_max_size { - let ids = std::mem::take(&mut pending_ids); - match redis::pipe() - .atomic() - .xack(&config.name, &config.group, &ids) - .xdel(&config.name, &ids) - .query_async::<_, redis::Value>(&mut connection) - .await - { - Ok(_) => { - redis_xack_inc(&config.name, ids.len()); - } - Err(e) => { - error!("Failed to acknowledge messages: {:?}", e); - } - } + pending.push(id); + let count = pending.len(); + if count >= config.xack_batch_max_size { + + let ids = std::mem::take(&mut pending); + xack_executor.push(AcknowledgeJob::Submit(ids)); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } }, - _ = &mut deadline => { - if !pending_ids.is_empty() { - let ids = std::mem::take(&mut pending_ids); - match redis::pipe() - .atomic() - .xack(&config.name, &config.group, &ids) - .xdel(&config.name, &ids) - .query_async::<_, redis::Value>(&mut connection) - .await - { - Ok(_) => { - redis_xack_inc(&config.name, ids.len()); - } - Err(e) => { - error!("Failed to acknowledge messages: {:?}", e); - } - } - } + _ = &mut deadline, if !pending.is_empty() => { + let ids = std::mem::take(&mut pending); + + xack_executor.push(AcknowledgeJob::Submit(ids)); + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); }, _ = &mut shutdown_rx => { + xack_executor.join_async().await; break; } } @@ -493,21 +526,19 @@ impl IngestStream { let opts = StreamReadOptions::default() .group(&config_read.group, &config_read.consumer) .count(config_read.batch_size) - .block(20); + .block(100); let result: RedisResult = connection_read .xread_options(&[&config_read.name], &[">"], &opts) .await; - info!( - "Read from stream name={} result={:?}", - config_read.name, result - ); match result { Ok(reply) => { + let count = reply.keys.len(); + info!("Reading and processing: count={:?}", count); + for StreamKey { key: _, ids } in reply.keys { for StreamId { id, map } in ids { - debug!("Reading and processing: id={} msg={:?}", id, map); executor.push(IngestStreamJob::Process((id, map))); } } @@ -769,12 +800,21 @@ async fn run_ack( let stream = Arc::clone(&stream); let mut connection = connection.clone(); async move { - redis::pipe() + match redis::pipe() .atomic() .xack(&stream.stream_name, &stream.group, &ids) .xdel(&stream.stream_name, &ids) - .query_async(&mut connection) - .await?; + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(_) => { + info!("Acknowledged and deleted idle messages: {:?}", ids); + redis_xack_inc(&stream.stream_name, ids.len()); + } + Err(e) => { + error!("Failed to acknowledge or delete idle messages: {:?}", e); + } + } redis_xack_inc(&stream.stream_name, ids.len()); Ok::<(), anyhow::Error>(()) } @@ -921,7 +961,6 @@ impl RedisStream { None => break, } - // read pending keys let StreamClaimReply { ids: pendings } = connection .xclaim( &stream.stream_name, @@ -1013,12 +1052,21 @@ impl RedisStream { let stream = Arc::clone(&stream); let mut connection = connection.clone(); async move { - redis::pipe() + match redis::pipe() .atomic() .xack(&stream.stream_name, &stream.group, &ids) .xdel(&stream.stream_name, &ids) - .query_async(&mut connection) - .await?; + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(info) => { + info!("Acknowledged and deleted idle messages: {:?}", info); + redis_xack_inc(&stream.stream_name, ids.len()); + } + Err(e) => { + error!("Failed to acknowledge or delete idle messages: {:?}", e); + } + } redis_xack_inc(&stream.stream_name, ids.len()); Ok::<(), anyhow::Error>(()) } diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index 7c45b68b7..a910b8d25 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -94,11 +94,11 @@ pub async fn index_and_fetch_mint_data( .map_err(|db_err| ProgramTransformerError::AssetIndexError(db_err.to_string()))?; Ok(Some(token)) } else { - warn!( - target: "Mint not found", - "Mint not found in 'tokens' table for mint {}", - bs58::encode(&mint_pubkey_vec).into_string() - ); + // warn!( + // target: "Mint not found", + // "Mint not found in 'tokens' table for mint {}", + // bs58::encode(&mint_pubkey_vec).into_string() + // ); Ok(None) } } @@ -133,11 +133,11 @@ async fn index_token_account_data( .await .map_err(|db_err| ProgramTransformerError::AssetIndexError(db_err.to_string()))?; } else { - warn!( - target: "Account not found", - "Token acc not found in 'owners' table for mint {}", - bs58::encode(&mint_pubkey_vec).into_string() - ); + // warn!( + // target: "Account not found", + // "Token acc not found in 'owners' table for mint {}", + // bs58::encode(&mint_pubkey_vec).into_string() + // ); } Ok(()) From 32f98522a83d1ae6f058f0658f34d36f8fc3d01c Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 22 Aug 2024 10:52:00 +0200 Subject: [PATCH 34/72] hack: disable pending processing while see why messages arent being deleted --- grpc-ingest/config-ingester.yml | 1 - grpc-ingest/src/redis.rs | 100 ++++++++++++++++---------------- 2 files changed, 50 insertions(+), 51 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index de4faa653..38404071b 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -27,7 +27,6 @@ program_transformer: accounts: name: ACCOUNTS max_concurrency: 10 - ack_buffer_size: 10 batch_size: 100 xack_batch_max_idle_ms: 1_000 xack_buffer_size: 10_000 diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 26c3d0014..471480fcc 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -15,8 +15,8 @@ use { redis::{ aio::MultiplexedConnection, streams::{ - StreamClaimOptions, StreamClaimReply, StreamId, StreamKey, StreamMaxlen, - StreamPendingCountReply, StreamReadOptions, StreamReadReply, + StreamClaimReply, StreamId, StreamKey, StreamMaxlen, StreamPendingCountReply, + StreamReadOptions, StreamReadReply, }, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, @@ -465,54 +465,54 @@ impl IngestStream { let mut shutdown_rx = read_shutdown_rx; - loop { - if shutdown_rx.try_recv().is_ok() { - debug!( - "Shutdown signal received, exiting prefetch loop name={}", - config_read.name - ); - break; - } - - if let Ok(pending) = redis::cmd("XPENDING") - .arg(&config_read.name) - .arg(&config_read.group) - .arg("-") - .arg("+") - .arg(config_read.batch_size) - .arg(&config_read.consumer) - .query_async::<_, StreamPendingCountReply>(&mut connection_read) - .await - { - if pending.ids.is_empty() { - debug!( - "No pending messages stream={} consumer={} group={}", - config_read.name, config_read.consumer, config_read.group - ); - break; - } - - let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); - let claim_opts = StreamClaimOptions::default(); - - let claimed: RedisResult = connection_read - .xclaim_options( - &config_read.name, - &config_read.group, - &config_read.consumer, - 20, - &ids, - claim_opts, - ) - .await; - - if let Ok(claimed) = claimed { - for StreamId { id, map } in claimed.ids { - executor.push(IngestStreamJob::Process((id, map))); - } - } - } - } + // loop { + // if shutdown_rx.try_recv().is_ok() { + // debug!( + // "Shutdown signal received, exiting prefetch loop name={}", + // config_read.name + // ); + // break; + // } + + // if let Ok(pending) = redis::cmd("XPENDING") + // .arg(&config_read.name) + // .arg(&config_read.group) + // .arg("-") + // .arg("+") + // .arg(config_read.batch_size) + // .arg(&config_read.consumer) + // .query_async::<_, StreamPendingCountReply>(&mut connection_read) + // .await + // { + // if pending.ids.is_empty() { + // debug!( + // "No pending messages stream={} consumer={} group={}", + // config_read.name, config_read.consumer, config_read.group + // ); + // break; + // } + + // let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); + // let claim_opts = StreamClaimOptions::default(); + + // let claimed: RedisResult = connection_read + // .xclaim_options( + // &config_read.name, + // &config_read.group, + // &config_read.consumer, + // 20, + // &ids, + // claim_opts, + // ) + // .await; + + // if let Ok(claimed) = claimed { + // for StreamId { id, map } in claimed.ids { + // executor.push(IngestStreamJob::Process((id, map))); + // } + // } + // } + // } loop { if shutdown_rx.try_recv().is_ok() { From bab4678e863ce20d67131c844c9450980617da5e Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 22 Aug 2024 13:24:04 +0200 Subject: [PATCH 35/72] fix: exit early if slot matches metadata json download and reindex is false. --- core/src/metadata_json.rs | 22 ++++++++++++++++--- grpc-ingest/src/config.rs | 16 -------------- grpc-ingest/src/download_metadata.rs | 2 +- grpc-ingest/src/main.rs | 1 - nft_ingester/src/tasks/common/mod.rs | 2 +- program_transformers/src/bubblegum/mint_v1.rs | 6 ++++- .../src/bubblegum/update_metadata.rs | 6 ++++- .../src/mpl_core_program/v1_asset.rs | 2 +- .../src/token_metadata/v1_asset.rs | 6 ++++- 9 files changed, 37 insertions(+), 26 deletions(-) diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs index eec14a46e..b66c81cc3 100644 --- a/core/src/metadata_json.rs +++ b/core/src/metadata_json.rs @@ -19,18 +19,20 @@ use { pub struct DownloadMetadataInfo { asset_data_id: Vec, uri: String, + slot: i64, } impl DownloadMetadataInfo { - pub fn new(asset_data_id: Vec, uri: String) -> Self { + pub fn new(asset_data_id: Vec, uri: String, slot: i64) -> Self { Self { asset_data_id, uri: uri.trim().replace('\0', ""), + slot, } } - pub fn into_inner(self) -> (Vec, String) { - (self.asset_data_id, self.uri) + pub fn into_inner(self) -> (Vec, String, i64) { + (self.asset_data_id, self.uri, self.slot) } } @@ -237,6 +239,20 @@ impl DownloadMetadata { &self, download_metadata_info: &DownloadMetadataInfo, ) -> Result<(), MetadataJsonTaskError> { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(self.pool.clone()); + + if let Some(asset_data) = + asset_data::Entity::find_by_id(download_metadata_info.asset_data_id.clone()) + .one(&conn) + .await? + { + if asset_data.slot_updated == download_metadata_info.slot + && asset_data.reindex == Some(false) + { + return Ok(()); + } + } + perform_metadata_json_task( self.client.clone(), self.pool.clone(), diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index f12b2cdef..d6ff8cc19 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -3,7 +3,6 @@ use { serde::{de, Deserialize}, std::{net::SocketAddr, path::Path, time::Duration}, tokio::fs, - tracing::warn, yellowstone_grpc_tools::config::{ deserialize_usize_str, ConfigGrpcRequestAccounts, ConfigGrpcRequestCommitment, ConfigGrpcRequestTransactions, @@ -260,21 +259,6 @@ pub struct ConfigIngester { pub transactions: ConfigIngestStream, } -impl ConfigIngester { - pub fn check(&self) { - let total_threads = self.accounts.max_concurrency - + self.transactions.max_concurrency - + self.download_metadata.stream_config.max_concurrency; - - if self.postgres.max_connections < total_threads { - warn!( - "postgres.max_connections ({}) should be more than the number of threads ({})", - self.postgres.max_connections, total_threads - ); - } - } -} - #[derive(Debug, Clone, Deserialize)] pub struct ConfigIngesterRedis { pub url: String, diff --git a/grpc-ingest/src/download_metadata.rs b/grpc-ingest/src/download_metadata.rs index 53eb9cdae..1e8e8a354 100644 --- a/grpc-ingest/src/download_metadata.rs +++ b/grpc-ingest/src/download_metadata.rs @@ -221,7 +221,7 @@ async fn download_metadata( data: serde_json::Value, timeout: Duration, ) -> anyhow::Result<(Vec, serde_json::Value)> { - let (id, uri) = serde_json::from_value::(data)?.into_inner(); + let (id, uri, _slot) = serde_json::from_value::(data)?.into_inner(); // Need to check for malicious sites ? let client = ClientBuilder::new().timeout(timeout).build()?; diff --git a/grpc-ingest/src/main.rs b/grpc-ingest/src/main.rs index 25beb0ecd..f467d3961 100644 --- a/grpc-ingest/src/main.rs +++ b/grpc-ingest/src/main.rs @@ -77,7 +77,6 @@ async fn main() -> anyhow::Result<()> { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; - config.check(); ingester::run_v2(config).await } ArgsAction::DownloadMetadata => { diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs index a74c650be..7d52439a9 100644 --- a/nft_ingester/src/tasks/common/mod.rs +++ b/nft_ingester/src/tasks/common/mod.rs @@ -25,7 +25,7 @@ pub fn create_download_metadata_notifier( 'static, Result<(), Box>, > { - let (asset_data_id, uri) = info.into_inner(); + let (asset_data_id, uri, _slot) = info.into_inner(); let task = DownloadMetadata { asset_data_id, uri, diff --git a/program_transformers/src/bubblegum/mint_v1.rs b/program_transformers/src/bubblegum/mint_v1.rs index fb179727a..bb2855b9a 100644 --- a/program_transformers/src/bubblegum/mint_v1.rs +++ b/program_transformers/src/bubblegum/mint_v1.rs @@ -207,7 +207,11 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(id_bytes.to_vec(), uri))) + Ok(Some(DownloadMetadataInfo::new( + id_bytes.to_vec(), + uri, + slot_i, + ))) } _ => Err(ProgramTransformerError::NotImplemented), }; diff --git a/program_transformers/src/bubblegum/update_metadata.rs b/program_transformers/src/bubblegum/update_metadata.rs index 702aa552c..2c55a3b8d 100644 --- a/program_transformers/src/bubblegum/update_metadata.rs +++ b/program_transformers/src/bubblegum/update_metadata.rs @@ -188,7 +188,11 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(id_bytes.to_vec(), uri))) + Ok(Some(DownloadMetadataInfo::new( + id_bytes.to_vec(), + uri, + slot_i, + ))) } _ => Err(ProgramTransformerError::NotImplemented), }; diff --git a/program_transformers/src/mpl_core_program/v1_asset.rs b/program_transformers/src/mpl_core_program/v1_asset.rs index 42e449286..d651739c0 100644 --- a/program_transformers/src/mpl_core_program/v1_asset.rs +++ b/program_transformers/src/mpl_core_program/v1_asset.rs @@ -461,7 +461,7 @@ pub async fn save_v1_asset( } // Otherwise return with info for background downloading. - Ok(Some(DownloadMetadataInfo::new(id_vec.clone(), uri))) + Ok(Some(DownloadMetadataInfo::new(id_vec.clone(), uri, slot_i))) } // Modify the JSON structure to remove the `Plugin` name and just display its data. diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index a910b8d25..446654f10 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -409,5 +409,9 @@ pub async fn save_v1_asset( return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(mint_pubkey_vec, uri))) + Ok(Some(DownloadMetadataInfo::new( + mint_pubkey_vec, + uri, + slot_i, + ))) } From 0c4f872c56c504e845d4c779b6cffe83d9eaea88 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 22 Aug 2024 21:17:33 +0200 Subject: [PATCH 36/72] fix: picking dangling pending messages before reading new --- backfill/src/worker/transaction.rs | 2 + grpc-ingest/src/ingester.rs | 5 +- grpc-ingest/src/redis.rs | 137 ++++++++++++++++------------- 3 files changed, 82 insertions(+), 62 deletions(-) diff --git a/backfill/src/worker/transaction.rs b/backfill/src/worker/transaction.rs index c047cffe4..3bb34b4b0 100644 --- a/backfill/src/worker/transaction.rs +++ b/backfill/src/worker/transaction.rs @@ -33,6 +33,7 @@ impl TryFrom for Pubkey { } } +#[derive(Debug)] pub struct FetchedEncodedTransactionWithStatusMeta(pub EncodedConfirmedTransactionWithStatusMeta); impl TryFrom for TransactionInfo { @@ -41,6 +42,7 @@ impl TryFrom for TransactionInfo { fn try_from( fetched_transaction: FetchedEncodedTransactionWithStatusMeta, ) -> Result { + tracing::info!("fetched transaction: {:?}", fetched_transaction); let mut account_keys = Vec::new(); let encoded_transaction_with_status_meta = fetched_transaction.0; diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 54391efc9..7af7bd44a 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -5,7 +5,7 @@ use { postgres::{create_pool as pg_create_pool, metrics_pgpool, report_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, - program_transformer_tasks_total_set, ProgramTransformerTaskStatusKind, + program_transformer_tasks_total_set, redis_xack_inc, ProgramTransformerTaskStatusKind, }, redis::{ metrics_xlen, IngestStream, ProgramTransformerInfo, RedisStream, RedisStreamMessage, @@ -58,7 +58,6 @@ fn download_metadata_notifier_v2( let mut connection = connection.clone(); let stream = stream.clone(); Box::pin(async move { - download_metadata_inserted_total_inc(); let info_bytes = serde_json::to_vec(&info)?; @@ -73,6 +72,8 @@ fn download_metadata_notifier_v2( .query_async(&mut connection) .await?; + redis_xack_inc(&stream, 1); + Ok(()) }) }, diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 471480fcc..268af0113 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -15,8 +15,8 @@ use { redis::{ aio::MultiplexedConnection, streams::{ - StreamClaimReply, StreamId, StreamKey, StreamMaxlen, StreamPendingCountReply, - StreamReadOptions, StreamReadReply, + StreamClaimOptions, StreamClaimReply, StreamId, StreamKey, StreamMaxlen, + StreamPendingCountReply, StreamReadOptions, StreamReadReply, }, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, @@ -298,7 +298,10 @@ impl<'a> .await { Ok(response) => { - info!("Acknowledged and deleted message: response={:?}", response); + debug!( + "Acknowledged and deleted message: stream={:?} response={:?} expected={:?}", + &config.name, response, count + ); redis_xack_inc(&config.name, count); } @@ -371,10 +374,10 @@ impl IngestStream { ) .await { - error!("Failed to create group: {:?}", e); + error!("redis=xgroup_create stream={} err={:?}", config.name, e); } else { debug!( - "Group created successfully: name={}, group={}, consumer={}", + "redis=xgroup_create stream={} group={} consumer={}", config.name, config.group, config.consumer ); } @@ -448,7 +451,7 @@ impl IngestStream { let streams = streams_report.clone(); if let Err(e) = report_xlen(connection, streams).await { - error!("Failed to report xlen: {:?}", e); + error!("redis=report_xlen err={:?}", e); } sleep(Duration::from_millis(100)).await; @@ -461,63 +464,71 @@ impl IngestStream { let (read_shutdown_tx, read_shutdown_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { - debug!("Starting read stream task name={}", config_read.name); + debug!( + "redis=read_stream stream={} Starting read stream task", + config_read.name + ); let mut shutdown_rx = read_shutdown_rx; - // loop { - // if shutdown_rx.try_recv().is_ok() { - // debug!( - // "Shutdown signal received, exiting prefetch loop name={}", - // config_read.name - // ); - // break; - // } - - // if let Ok(pending) = redis::cmd("XPENDING") - // .arg(&config_read.name) - // .arg(&config_read.group) - // .arg("-") - // .arg("+") - // .arg(config_read.batch_size) - // .arg(&config_read.consumer) - // .query_async::<_, StreamPendingCountReply>(&mut connection_read) - // .await - // { - // if pending.ids.is_empty() { - // debug!( - // "No pending messages stream={} consumer={} group={}", - // config_read.name, config_read.consumer, config_read.group - // ); - // break; - // } - - // let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); - // let claim_opts = StreamClaimOptions::default(); - - // let claimed: RedisResult = connection_read - // .xclaim_options( - // &config_read.name, - // &config_read.group, - // &config_read.consumer, - // 20, - // &ids, - // claim_opts, - // ) - // .await; - - // if let Ok(claimed) = claimed { - // for StreamId { id, map } in claimed.ids { - // executor.push(IngestStreamJob::Process((id, map))); - // } - // } - // } - // } + let mut start = "-".to_owned(); + loop { + if shutdown_rx.try_recv().is_ok() { + debug!( + "redis=read_stream stream={} Shutdown signal received, exiting prefetch loop", + config_read.name + ); + break; + } + + if let Ok(pending) = redis::cmd("XPENDING") + .arg(&config_read.name) + .arg(&config_read.group) + .arg(&start) + .arg("+") + .arg(config_read.batch_size) + .arg(&config_read.consumer) + .query_async::<_, StreamPendingCountReply>(&mut connection_read) + .await + { + if pending.ids.is_empty() { + debug!( + "redis=XPENDING stream={} consumer={} group={} No pending messages", + config_read.name, config_read.consumer, config_read.group + ); + break; + } + + let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); + let claim_opts = StreamClaimOptions::default(); + + let claimed: RedisResult = connection_read + .xclaim_options( + &config_read.name, + &config_read.group, + &config_read.consumer, + 20, + &ids, + claim_opts, + ) + .await; + + if let Ok(claimed) = claimed { + for StreamId { id, map } in claimed.ids { + executor.push(IngestStreamJob::Process((id, map))); + } + } + + if let Some(last_id) = pending.ids.last() { + start = last_id.id.clone(); + } + } + } loop { if shutdown_rx.try_recv().is_ok() { debug!( - "Shutdown signal received, exiting read loop name={}", + "redis=read_stream stream={} Shutdown signal received, exiting read loop", config_read.name ); break; @@ -535,7 +546,10 @@ impl IngestStream { match result { Ok(reply) => { let count = reply.keys.len(); - info!("Reading and processing: count={:?}", count); + debug!( + "redis=xread stream={:?} count={:?}", + &config_read.name, count + ); for StreamKey { key: _, ids } in reply.keys { for StreamId { id, map } in ids { @@ -544,7 +558,7 @@ impl IngestStream { } } Err(err) => { - error!("Error reading from stream: {:?}", err); + error!("redis=xread stream={:?} err={:?}", &config_read.name, err); } } } @@ -552,11 +566,14 @@ impl IngestStream { let control = tokio::spawn(async move { let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = shutdown_rx; - debug!("Starting ingest stream name={}", config.name); + debug!( + "redis=ingest_stream stream={} Starting ingest stream", + config.name + ); tokio::select! { _ = &mut shutdown_rx => { - info!("Shut down ingest stream name={}", config.name); + info!("redis=ingest_stream stream={} Shut down ingest stream", config.name); let _ = read_shutdown_tx.send(()); let _ = ack_shutdown_tx.send(()); From eb75ca35b55a5b16c8ffc6b5b47e556d25314eb3 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 23 Aug 2024 12:23:54 +0200 Subject: [PATCH 37/72] fix: ingest stream shutdown and requesting snapshots --- Cargo.lock | 511 ++++++++++++++++++------------------ Cargo.toml | 20 +- grpc-ingest/src/config.rs | 7 + grpc-ingest/src/grpc.rs | 1 + grpc-ingest/src/ingester.rs | 22 +- grpc-ingest/src/redis.rs | 270 ++++++++++--------- 6 files changed, 427 insertions(+), 404 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 396e8e69c..0f6b76380 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "getrandom 0.2.10", @@ -785,9 +785,9 @@ checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", @@ -943,9 +943,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -1005,9 +1005,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde", ] @@ -1143,6 +1143,16 @@ dependencies = [ "hashbrown 0.13.2", ] +[[package]] +name = "borsh" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +dependencies = [ + "borsh-derive 1.5.1", + "cfg_aliases", +] + [[package]] name = "borsh-derive" version = "0.9.3" @@ -1169,6 +1179,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +dependencies = [ + "once_cell", + "proc-macro-crate 3.1.0", + "proc-macro2", + "quote", + "syn 2.0.66", + "syn_derive", +] + [[package]] name = "borsh-derive-internal" version = "0.9.3" @@ -1385,6 +1409,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.31" @@ -1547,15 +1577,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -1964,16 +1994,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if", - "num_cpus", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -2276,18 +2296,18 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", @@ -2493,9 +2513,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -2529,9 +2549,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2560,9 +2580,9 @@ checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2829,7 +2849,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", ] [[package]] @@ -2838,7 +2858,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", "allocator-api2", ] @@ -3020,7 +3040,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", ] @@ -3081,9 +3101,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -3272,9 +3292,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -3439,9 +3459,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libsecp256k1" @@ -4125,7 +4145,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -4391,9 +4411,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "percentage" @@ -4546,9 +4566,9 @@ dependencies = [ [[package]] name = "plerkle_serialization" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f832646491065468aa8e222b47d41dd5250e4be7866725bef5f0d31c64538f5f" +checksum = "69341a546676367be06201860e72bc7ebb8f228e7dcd1999aae715207ca0a816" dependencies = [ "bs58 0.4.0", "chrono", @@ -4660,6 +4680,15 @@ dependencies = [ "toml_edit 0.19.15", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.0", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4733,6 +4762,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "project-root" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bccbff07d5ed689c4087d20d7307a52ab6141edeedf487c3876a55b86cf63df" + [[package]] name = "prometheus" version = "0.13.3" @@ -4877,7 +4912,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.12", "thiserror", "tokio", "tracing", @@ -4893,7 +4928,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.12", "rustls-native-certs", "slab", "thiserror", @@ -5049,9 +5084,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -5059,9 +5094,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -5219,12 +5254,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -5243,11 +5278,12 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -5447,7 +5483,7 @@ version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.10", @@ -5468,12 +5504,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.2", "rustls-webpki", "sct", ] @@ -5496,17 +5532,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.2", + "untrusted 0.9.0", ] [[package]] @@ -5803,9 +5839,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -5821,9 +5857,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" dependencies = [ "serde", ] @@ -5852,12 +5888,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "indexmap 2.3.0", "itoa", + "memchr", "ryu", "serde", ] @@ -5899,7 +5936,7 @@ version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -5954,7 +5991,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ - "dashmap 5.5.3", + "dashmap", "futures", "lazy_static", "log", @@ -6091,6 +6128,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "sized-chunks" version = "0.6.5" @@ -6112,9 +6155,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -6154,12 +6197,12 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" +checksum = "b4185d569c062983fc2a618ae4ee6fe1a139b36bce7a25045647c49bf0020a53" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "bv", @@ -6179,9 +6222,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" +checksum = "c817832e71886dbea877d1aa911c9ce2e984a39081bb56ee30d4c835567827a6" dependencies = [ "chrono", "clap 2.34.0", @@ -6196,13 +6239,13 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" +checksum = "7fa9cc6e8e59adf70acbf5cac21342ae8b5e41cbf05519fe5f6287e84ab40f63" dependencies = [ "async-trait", "bincode", - "dashmap 4.0.2", + "dashmap", "futures", "futures-util", "indexmap 2.3.0", @@ -6229,9 +6272,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" +checksum = "d02fb29934427f1487d2149fe8bcb405306729b2f22a2ad616bb8ffd024cee7b" dependencies = [ "bincode", "chrono", @@ -6243,9 +6286,9 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" +checksum = "d8e5a2e26448b3e04ce673794994ff27f3972ec8a806c224eccc02e09f751ca5" dependencies = [ "async-trait", "bincode", @@ -6265,17 +6308,13 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" +checksum = "20a6ef2db80dceb124b7bf81cca3300804bf427d2711973fc3df450ed7dfb26d" dependencies = [ - "ahash 0.8.5", - "blake3", "block-buffer 0.10.4", "bs58 0.4.0", "bv", - "byteorder", - "cc", "either", "generic-array", "im", @@ -6286,7 +6325,6 @@ dependencies = [ "serde", "serde_bytes", "serde_derive", - "serde_json", "sha2 0.10.8", "solana-frozen-abi-macro", "subtle", @@ -6295,9 +6333,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" +checksum = "70088de7d4067d19a7455609e2b393e6086bd847bb39c4d2bf234fc14827ef9e" dependencies = [ "proc-macro2", "quote", @@ -6307,9 +6345,9 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-interface" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633425dc9409c6d3f019658b90bb1ad53747ed55acd45e0a18f58b95a0b89e5" +checksum = "63f1835fe954e305097c83b4ce8548e675647eef6a3ea9ec13dbfe2d6b869398" dependencies = [ "log", "solana-sdk", @@ -6319,9 +6357,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" +checksum = "b129da15193f26db62d62ae6bb9f72361f361bcdc36054be3ab8bc04cc7a4f31" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -6330,9 +6368,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" +checksum = "6d195b73093a4964ba6b5943418054a5fcbba23eafdd0842fd973fcceac1a967" dependencies = [ "log", "solana-sdk", @@ -6340,9 +6378,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" +checksum = "fe7b06860ffbf4cf4714182e1b7eb00eb3ff0bcc9cff615d05e01e488923883c" dependencies = [ "crossbeam-channel", "gethostname", @@ -6355,9 +6393,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" +checksum = "9400b50b8439868a99b5fa2d961d74e37b7a6c1d5865759d0b1c906c2ad6b2a9" dependencies = [ "bincode", "clap 3.2.25", @@ -6377,11 +6415,11 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" +checksum = "b01a386e852df67031195094628851b8d239dd71fe17b721c3993277e68cb3ab" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", "bincode", "bv", "caps", @@ -6406,20 +6444,21 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" +checksum = "fb2b2c8babfae4cace1a25b6efa00418f3acd852cf55d7cecc0360d3c5050479" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.6.0", "blake3", "borsh 0.10.3", "borsh 0.9.3", + "borsh 1.5.1", "bs58 0.4.0", "bv", "bytemuck", @@ -6437,7 +6476,7 @@ dependencies = [ "log", "memoffset 0.9.0", "num-bigint 0.4.4", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "parking_lot 0.12.1", "rand 0.8.5", @@ -6460,18 +6499,18 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" +checksum = "0444f9440f4459d377c41470b2eb48b527def81f3052b7a121f6aa8c7350cc52" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bincode", "eager", "enum-iterator", "itertools 0.10.5", "libc", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "percentage", "rand 0.8.5", @@ -6488,9 +6527,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" +checksum = "0ee4a39e41e789b6f100c97d9f40c1d08381bf6e3d0e351065e542091cddb039" dependencies = [ "crossbeam-channel", "futures-util", @@ -6513,9 +6552,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" +checksum = "baad755c76ee0aab8890f0ef873e61b8b3012c523d33bfa5b062fe9be8cef370" dependencies = [ "async-mutex", "async-trait", @@ -6526,7 +6565,7 @@ dependencies = [ "quinn", "quinn-proto", "rcgen", - "rustls 0.21.7", + "rustls 0.21.12", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -6540,9 +6579,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" +checksum = "c1c2a0ccb0be7ca79e8ff0d7c786bce586433a5687ffbea522453d0b41c4bf4a" dependencies = [ "lazy_static", "num_cpus", @@ -6550,14 +6589,14 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" +checksum = "3d042a812537e3507e1c163c7573fc04c96e12d3eba512e3fe74c7393229fa39" dependencies = [ "console", "dialoguer", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "parking_lot 0.12.1", "qstring", @@ -6569,12 +6608,12 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" +checksum = "3c6f5560283bd0a6833d1bd816299785058a870fff51b0df399fdb3ce92c8484" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "indicatif", @@ -6595,11 +6634,11 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" +checksum = "2e4ca77f89caa9071acadb1eed19c28a6691fd63d0563ed927c96bf734cf1c9c" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", "reqwest", @@ -6617,9 +6656,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" +checksum = "42a6ea9ad81d63f18fb8b3a9b39643cc43eaf909199d67037e724562301d1df7" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -6630,15 +6669,15 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" +checksum = "b5e0f0def5c5af07f53d321cea7b104487b522cfff77c3cae3da361bfe956e9e" dependencies = [ "assert_matches", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", - "borsh 0.10.3", + "bitflags 2.6.0", + "borsh 1.5.1", "bs58 0.4.0", "bytemuck", "byteorder", @@ -6655,9 +6694,9 @@ dependencies = [ "libsecp256k1", "log", "memmap2", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", - "num_enum 0.6.1", + "num_enum 0.7.2", "pbkdf2 0.11.0", "qstring", "qualifier_attr", @@ -6672,6 +6711,7 @@ dependencies = [ "serde_with 2.3.3", "sha2 0.10.8", "sha3 0.10.8", + "siphasher", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-logger", @@ -6684,9 +6724,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" +checksum = "c55c196c8050834c391a34b58e3c9fd86b15452ef1feeeafa1dbeb9d2291dfec" dependencies = [ "bs58 0.4.0", "proc-macro2", @@ -6703,9 +6743,9 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" +checksum = "749720d82c5f31f7ec326da1e0baac098201de70f0874719172a55309433b449" dependencies = [ "async-channel 1.9.0", "bytes", @@ -6724,7 +6764,8 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rcgen", - "rustls 0.21.7", + "rustls 0.21.12", + "smallvec", "solana-metrics", "solana-perf", "solana-sdk", @@ -6735,9 +6776,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" +checksum = "84535de1253afb6ccc4ae6852eb013ca734c439a902ec5e4684b90ed649a37c2" dependencies = [ "bincode", "log", @@ -6750,9 +6791,9 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" +checksum = "3ff514462bb715aaea9bc5c0ee60f83ab3f91e04279337c6b07d054153b616dc" dependencies = [ "async-trait", "bincode", @@ -6774,12 +6815,12 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" +checksum = "670e387049812d42bdc8fcc4ff75452ff3cb00657af979a90f55f6d37dba9dd9" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "borsh 0.10.3", "bs58 0.4.0", @@ -6799,9 +6840,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" +checksum = "11183dae826f942ebd0401712c8a52367a4a6312f1cd325f304cd9551226fc8b" dependencies = [ "async-trait", "solana-connection-cache", @@ -6814,9 +6855,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" +checksum = "8e8d518e61ce22c812df23d9c61ab9bcbef4df3e3d3dcaa74a999625f11bcf07" dependencies = [ "log", "rustc_version", @@ -6830,13 +6871,13 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" +checksum = "5743503143fb2259c41a973a78e9aeeb8e21f1b03543c3bb85449926ea692719" dependencies = [ "bincode", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "rustc_version", "serde", @@ -6852,12 +6893,12 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.28" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" +checksum = "57ee07fa523b4cfcff68de774db7aa87d2da2c4357155a90bacd9a0a0af70a99" dependencies = [ "aes-gcm-siv", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bytemuck", "byteorder", @@ -6866,7 +6907,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "merlin", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "rand 0.7.3", "serde", @@ -6881,9 +6922,9 @@ dependencies = [ [[package]] name = "solana_rbpf" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d457cc2ba742c120492a64b7fa60e22c575e891f6b55039f4d736568fb112a3" +checksum = "da5d083187e3b3f453e140f292c09186881da8a02a7b5e27f645ee26de3d9cc5" dependencies = [ "byteorder", "combine 3.8.1", @@ -7018,7 +7059,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "borsh 0.10.3", "bytemuck", "serde", @@ -7357,6 +7398,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -7441,18 +7494,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", @@ -7633,7 +7686,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.12", "tokio", ] @@ -7656,7 +7709,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.7", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", "tungstenite", @@ -7755,7 +7808,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.7", "bytes", "flate2", "h2", @@ -7766,7 +7819,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "rustls 0.21.7", + "rustls 0.21.12", "rustls-native-certs", "rustls-pemfile", "tokio", @@ -8022,7 +8075,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.7", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -8179,9 +8232,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -8305,19 +8358,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", @@ -8342,9 +8396,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8352,9 +8406,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", @@ -8365,9 +8419,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-sys" @@ -8486,15 +8540,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -8513,21 +8558,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -8559,12 +8589,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.5", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8577,12 +8601,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -8595,12 +8613,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -8619,12 +8631,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -8637,12 +8643,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -8655,12 +8655,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -8673,12 +8667,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -8769,8 +8757,8 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.15.0+solana.1.17.28" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +version = "1.15.3+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" dependencies = [ "bytes", "futures", @@ -8782,8 +8770,8 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "1.14.0+solana.1.17.28" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +version = "1.14.2+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" dependencies = [ "anyhow", "bincode", @@ -8798,8 +8786,8 @@ dependencies = [ [[package]] name = "yellowstone-grpc-tools" -version = "1.0.0-rc.11+solana.1.17.28" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.14.1+solana.1.17.28#8fb667501cdcee0f50d8edbf5c382c97f6b4bf07" +version = "1.0.0-rc.11+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" dependencies = [ "anyhow", "async-trait", @@ -8811,6 +8799,7 @@ dependencies = [ "hyper", "json5", "lazy_static", + "project-root", "prometheus", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index ffeffdf61..8a77a8473 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,12 +109,12 @@ serde = "1.0.137" serde_json = "1.0.81" serde_yaml = "0.9.34" serial_test = "2.0.0" -solana-account-decoder = "~1.17" -solana-client = "~1.17" -solana-geyser-plugin-interface = "~1.17" -solana-program = "~1.17" -solana-sdk = "~1.17" -solana-transaction-status = "~1.17" +solana-account-decoder = "~1.18" +solana-client = "~1.18" +solana-geyser-plugin-interface = "~1.18" +solana-program = "~1.18" +solana-sdk = "~1.18" +solana-transaction-status = "~1.18" solana-zk-token-sdk = "1.17.16" spl-account-compression = "0.3.0" spl-associated-token-account = ">= 1.1.3, < 3.0" @@ -132,7 +132,7 @@ tokio = "1.30.0" tokio-stream = "0.1.14" topograph = { version = "0.4.0", features = [ "tokio", -], git = "https://github.com/ray-kast/topograph", sha = "5449d34ff726c2ab9f4d3a59a7070ae7bdc76027", branch = "v0.4-wip" } +], git = "https://github.com/ray-kast/topograph", branch = "v0.4-wip" } tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" @@ -143,9 +143,9 @@ url = "2.3.1" vergen = "8.2.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" -yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin -yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28" } # tag is geyser plugin -yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.14.1+solana.1.17.28", default-features = false } # tag is geyser plugin +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22", default-features = false } # tag is geyser plugin [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index d6ff8cc19..ebd47ea5b 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -124,6 +124,9 @@ pub struct ConfigGrpc { pub geyser_endpoint: String, + #[serde(default = "ConfigGrpc::default_request_snapshot")] + pub request_snapshot: bool, + #[serde( default = "ConfigGrpc::default_geyser_update_message_buffer_size", deserialize_with = "deserialize_usize_str" @@ -146,6 +149,10 @@ pub struct ConfigGrpc { } impl ConfigGrpc { + pub const fn default_request_snapshot() -> bool { + false + } + pub const fn default_max_concurrency() -> usize { 10 } diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 41d94573d..5e3172794 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -147,6 +147,7 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { .x_token(config.x_token.clone())? .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10)) + .set_x_request_snapshot(config.request_snapshot)? .connect() .await .context("failed to connect to gRPC")?; diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 7af7bd44a..53f34ef1d 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -163,19 +163,15 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { let mut shutdown = create_shutdown()?; - let reporting_pool = pool.clone(); - - tokio::spawn(async move { - loop { - report_pgpool(reporting_pool.clone()); - - sleep(Duration::from_millis(100)).await; - } - }); - - tokio::select! { - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); + loop { + tokio::select! { + _ = sleep(Duration::from_millis(100)) => { + report_pgpool(pool.clone()); + } + Some(signal) = shutdown.next() => { + warn!("{signal} received, waiting spawned tasks..."); + break; + } } } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 268af0113..26485f08f 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -63,7 +63,6 @@ impl IngestStreamStop { let _ = self.shutdown_tx.send(()); self.control.await?; - // self.executor.join_async().await; Ok(()) } @@ -314,7 +313,7 @@ impl<'a> } pub struct IngestStream { - config: ConfigIngestStream, + config: Arc, connection: Option, handler: Option>, } @@ -322,14 +321,14 @@ pub struct IngestStream { impl IngestStream { pub fn build() -> Self { Self { - config: ConfigIngestStream::default(), + config: Arc::new(ConfigIngestStream::default()), connection: None, handler: None, } } pub fn config(mut self, config: ConfigIngestStream) -> Self { - self.config = config; + self.config = Arc::new(config); self } @@ -349,8 +348,58 @@ impl IngestStream { self } + async fn pending( + &self, + connection: &mut MultiplexedConnection, + start: &str, + ) -> RedisResult> { + let config = Arc::clone(&self.config); + + let pending = redis::cmd("XPENDING") + .arg(&config.name) + .arg(&config.group) + .arg(start) + .arg("+") + .arg(config.batch_size) + .arg(&config.consumer) + .query_async::<_, StreamPendingCountReply>(connection) + .await?; + let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); + let opts = StreamClaimOptions::default(); + + let claimed: StreamClaimReply = connection + .xclaim_options( + &config.name, + &config.group, + &config.consumer, + 100, + &ids, + opts, + ) + .await?; + + if claimed.ids.is_empty() { + return Ok(None); + } + + Ok(Some(claimed)) + } + + async fn read(&self, connection: &mut MultiplexedConnection) -> RedisResult { + let config = &self.config; + + let opts = StreamReadOptions::default() + .group(&config.group, &config.consumer) + .count(config.batch_size) + .block(100); + + connection + .xread_options(&[&config.name], &[">"], &opts) + .await + } + pub fn start(mut self) -> anyhow::Result { - let config = Arc::new(self.config); + let config = Arc::clone(&self.config); let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); @@ -396,7 +445,7 @@ impl IngestStream { connection: connection.clone(), })?; - tokio::spawn({ + let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); @@ -442,142 +491,123 @@ impl IngestStream { ack_sender: ack_tx.clone(), })?; - let connection_report = connection.clone(); - let streams_report = vec![config.name.clone()]; + let report = vec![config.name.clone()]; - tokio::spawn(async move { - loop { - let connection = connection_report.clone(); - let streams = streams_report.clone(); - - if let Err(e) = report_xlen(connection, streams).await { - error!("redis=report_xlen err={:?}", e); - } + let control = tokio::spawn({ + let mut connection = connection.clone(); + async move { + let config = Arc::clone(&config); - sleep(Duration::from_millis(100)).await; - } - }); + debug!( + "redis=read_stream stream={} Starting read stream task", + config.name + ); - let config_read = Arc::clone(&config); - let mut connection_read = connection.clone(); + let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = shutdown_rx; - let (read_shutdown_tx, read_shutdown_rx) = tokio::sync::oneshot::channel(); + let mut start = "-".to_owned(); - tokio::spawn(async move { - debug!( - "redis=read_stream stream={} Starting read stream task", - config_read.name - ); + loop { + tokio::select! { - let mut shutdown_rx = read_shutdown_rx; + _ = &mut shutdown_rx => { + debug!( + "redis=read_stream stream={} Shutdown signal received, exiting loops", + config.name + ); + break; + }, + _ = sleep(Duration::from_millis(100)) => { + let connection = connection.clone(); - let mut start = "-".to_owned(); - loop { - if shutdown_rx.try_recv().is_ok() { - debug!( - "redis=read_stream stream={} Shutdown signal received, exiting prefetch loop", - config_read.name - ); - break; - } + if let Err(e) = report_xlen(connection, report.clone()).await { + error!("redis=report_xlen err={:?}", e); + } - if let Ok(pending) = redis::cmd("XPENDING") - .arg(&config_read.name) - .arg(&config_read.group) - .arg(&start) - .arg("+") - .arg(config_read.batch_size) - .arg(&config_read.consumer) - .query_async::<_, StreamPendingCountReply>(&mut connection_read) - .await - { - if pending.ids.is_empty() { - debug!( - "redis=XPENDING stream={} consumer={} group={} No pending messages", - config_read.name, config_read.consumer, config_read.group - ); - break; - } + debug!( + "redis=read_stream stream={} msg=waiting for pending messages", + config.name + ); + }, + claimed = self.pending(&mut connection, &start) => { + if let Ok(Some(claimed)) = claimed { + let ids = claimed.ids.clone(); + let ids: Vec<&str> = ids.iter().map(|info| info.id.as_str()).collect(); - let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); - let claim_opts = StreamClaimOptions::default(); - - let claimed: RedisResult = connection_read - .xclaim_options( - &config_read.name, - &config_read.group, - &config_read.consumer, - 20, - &ids, - claim_opts, - ) - .await; + for StreamId { id, map } in claimed.ids.into_iter() { + executor.push(IngestStreamJob::Process((id, map))); + } - if let Ok(claimed) = claimed { - for StreamId { id, map } in claimed.ids { - executor.push(IngestStreamJob::Process((id, map))); - } - } - if let Some(last_id) = pending.ids.last() { - start = last_id.id.clone(); + if let Some(last) = ids.last() { + start = last.to_string(); + } + } else { + break; + } + }, } } - } - loop { - if shutdown_rx.try_recv().is_ok() { - debug!( - "redis=read_stream stream={} Shutdown signal received, exiting read loop", - config_read.name - ); - break; - } + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!( + "redis=read_stream stream={} Shutdown signal received, exiting read loop", + config.name + ); + break; + }, + _ = sleep(Duration::from_millis(100)) => { + let connection = connection.clone(); + + if let Err(e) = report_xlen(connection, report.clone()).await { + error!("redis=report_xlen err={:?}", e); + } - let opts = StreamReadOptions::default() - .group(&config_read.group, &config_read.consumer) - .count(config_read.batch_size) - .block(100); - - let result: RedisResult = connection_read - .xread_options(&[&config_read.name], &[">"], &opts) - .await; - - match result { - Ok(reply) => { - let count = reply.keys.len(); - debug!( - "redis=xread stream={:?} count={:?}", - &config_read.name, count - ); - - for StreamKey { key: _, ids } in reply.keys { - for StreamId { id, map } in ids { - executor.push(IngestStreamJob::Process((id, map))); + debug!( + "redis=read_stream stream={} msg=waiting for pending messages", + config.name + ); + }, + result = self.read(&mut connection) => { + match result { + Ok(reply) => { + let count = reply.keys.len(); + debug!( + "redis=xread stream={:?} count={:?}", + &config.name, count + ); + + for StreamKey { key: _, ids } in reply.keys { + for StreamId { id, map } in ids { + executor.push(IngestStreamJob::Process((id, map))); + } + } + } + Err(err) => { + error!("redis=xread stream={:?} err={:?}", &config.name, err); + } } } } - Err(err) => { - error!("redis=xread stream={:?} err={:?}", &config_read.name, err); - } } - } - }); - let control = tokio::spawn(async move { - let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = shutdown_rx; - debug!( - "redis=ingest_stream stream={} Starting ingest stream", - config.name - ); + debug!("stream={} msg=start shut down ingest stream", config.name); - tokio::select! { - _ = &mut shutdown_rx => { - info!("redis=ingest_stream stream={} Shut down ingest stream", config.name); + executor.join_async().await; - let _ = read_shutdown_tx.send(()); - let _ = ack_shutdown_tx.send(()); + debug!("stream={} msg=shut down executor", config.name); + + if let Err(e) = ack_shutdown_tx.send(()) { + error!("Failed to send shutdown signal: {:?}", e); + } + + if let Err(e) = ack.await { + error!("Error during shutdown: {:?}", e); } + + debug!("stream={} msg=shut down ack", config.name); } }); @@ -825,7 +855,7 @@ async fn run_ack( .await { Ok(_) => { - info!("Acknowledged and deleted idle messages: {:?}", ids); + debug!("Acknowledged and deleted idle messages: {:?}", ids); redis_xack_inc(&stream.stream_name, ids.len()); } Err(e) => { From 32130803b915033e45de1e51c370a5812a4865bb Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 23 Aug 2024 20:28:06 +0200 Subject: [PATCH 38/72] fix: processing pending. add metrics for total workers. --- grpc-ingest/src/grpc.rs | 7 +++- grpc-ingest/src/prom.rs | 18 +++++++++ grpc-ingest/src/redis.rs | 83 +++++++++++++++++++++++++--------------- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 5e3172794..5212a589c 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -20,7 +20,7 @@ use { prelude::*, AsyncHandler, }, - tracing::{debug, warn}, + tracing::{debug, info, warn}, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::{SubscribeRequest, SubscribeUpdate}, @@ -142,6 +142,11 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { ..Default::default() }; + info!( + "cmd=grpc2redis request_snapshot={:?}", + config.request_snapshot + ); + let mut dragon_mouth_client = GeyserGrpcClient::build_from_shared(config.geyser_endpoint.clone())? .x_token(config.x_token.clone())? diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 6beab4614..83b51011c 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -52,6 +52,11 @@ lazy_static::lazy_static! { static ref DOWNLOAD_METADATA_INSERTED_TOTAL: IntCounter = IntCounter::new( "download_metadata_inserted_total", "Total number of inserted tasks for download metadata" ).unwrap(); + + static ref INGEST_TASKS_TOTAL: IntGaugeVec = IntGaugeVec::new( + Opts::new("ingest_tasks_total", "Number of tasks spawned for ingest"), + &["stream"] + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -72,6 +77,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(PROGRAM_TRANSFORMER_TASKS_TOTAL); register!(PROGRAM_TRANSFORMER_TASK_STATUS); register!(DOWNLOAD_METADATA_INSERTED_TOTAL); + register!(INGEST_TASKS_TOTAL); VERSION .with_label_values(&[ @@ -160,6 +166,18 @@ pub fn program_transformer_tasks_total_set(size: usize) { PROGRAM_TRANSFORMER_TASKS_TOTAL.set(size as i64) } +pub fn ingest_tasks_total_inc(stream: &str) { + INGEST_TASKS_TOTAL.with_label_values(&[stream]).inc() +} + +pub fn ingest_tasks_total_dec(stream: &str) { + INGEST_TASKS_TOTAL.with_label_values(&[stream]).dec() +} + +pub fn ingest_tasks_reset(stream: &str) { + INGEST_TASKS_TOTAL.with_label_values(&[stream]).set(0) +} + #[derive(Debug, Clone, Copy)] pub enum ProgramTransformerTaskStatusKind { Success, diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 26485f08f..1cd0d564b 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -5,6 +5,7 @@ use { REDIS_STREAM_DATA_KEY, }, prom::{ + ingest_tasks_reset, ingest_tasks_total_dec, ingest_tasks_total_inc, program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, ProgramTransformerTaskStatusKind, }, @@ -76,6 +77,7 @@ type HandlerFn = dyn Fn(HashMap) -> BoxFuture<'static, Resul pub struct IngestStreamHandler { handler: Arc, ack_sender: tokio::sync::mpsc::Sender, + config: Arc, } impl<'a> @@ -91,7 +93,9 @@ impl<'a> ) -> impl futures::Future + Send { let handler = Arc::clone(&self.handler); let ack_sender = self.ack_sender.clone(); + let config = Arc::clone(&self.config); + ingest_tasks_total_inc(&config.name); async move { match job { IngestStreamJob::Process((id, msg)) => { @@ -117,6 +121,8 @@ impl<'a> if let Err(e) = ack_sender.send(id).await { error!("Failed to send ack id to channel: {:?}", e); } + + ingest_tasks_total_dec(&config.name); } } } @@ -484,14 +490,49 @@ impl IngestStream { let handler = self.handler.take().expect("Handler is required"); + ingest_tasks_reset(&config.name); + let executor = Executor::builder(Nonblock(Tokio)) .max_concurrency(Some(config.max_concurrency)) .build_async(IngestStreamHandler { handler, ack_sender: ack_tx.clone(), + config: Arc::clone(&config), })?; - let report = vec![config.name.clone()]; + let labels = vec![config.name.clone()]; + let (report_shutdown_tx, report_shutdown_rx) = tokio::sync::oneshot::channel(); + let report_thread = tokio::spawn({ + let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = report_shutdown_rx; + let connection = connection.clone(); + let config = Arc::clone(&config); + + async move { + let config = Arc::clone(&config); + + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!( + "redis=report_thread stream={} Shutdown signal received, exiting report loop", + config.name + ); + break; + }, + _ = sleep(Duration::from_millis(100)) => { + if let Err(e) = report_xlen(connection.clone(), labels.clone()).await { + error!("redis=report_xlen err={:?}", e); + } + + debug!( + "redis=report_thread stream={} msg=waiting for pending messages", + config.name + ); + }, + } + } + } + }); let control = tokio::spawn({ let mut connection = connection.clone(); @@ -509,7 +550,6 @@ impl IngestStream { loop { tokio::select! { - _ = &mut shutdown_rx => { debug!( "redis=read_stream stream={} Shutdown signal received, exiting loops", @@ -517,19 +557,8 @@ impl IngestStream { ); break; }, - _ = sleep(Duration::from_millis(100)) => { - let connection = connection.clone(); - - if let Err(e) = report_xlen(connection, report.clone()).await { - error!("redis=report_xlen err={:?}", e); - } - - debug!( - "redis=read_stream stream={} msg=waiting for pending messages", - config.name - ); - }, claimed = self.pending(&mut connection, &start) => { + info!("redis=claimed stream={} claimed={:?}", config.name, claimed); if let Ok(Some(claimed)) = claimed { let ids = claimed.ids.clone(); let ids: Vec<&str> = ids.iter().map(|info| info.id.as_str()).collect(); @@ -558,18 +587,6 @@ impl IngestStream { ); break; }, - _ = sleep(Duration::from_millis(100)) => { - let connection = connection.clone(); - - if let Err(e) = report_xlen(connection, report.clone()).await { - error!("redis=report_xlen err={:?}", e); - } - - debug!( - "redis=read_stream stream={} msg=waiting for pending messages", - config.name - ); - }, result = self.read(&mut connection) => { match result { Ok(reply) => { @@ -597,17 +614,23 @@ impl IngestStream { executor.join_async().await; - debug!("stream={} msg=shut down executor", config.name); + if let Err(e) = report_shutdown_tx.send(()) { + error!("Failed to send report shutdown signal: {:?}", e); + } if let Err(e) = ack_shutdown_tx.send(()) { - error!("Failed to send shutdown signal: {:?}", e); + error!("Failed to send ack shutdown signal: {:?}", e); } if let Err(e) = ack.await { - error!("Error during shutdown: {:?}", e); + error!("Error during ack shutdown: {:?}", e); + } + + if let Err(e) = report_thread.await { + error!("Error during report thread shutdown: {:?}", e); } - debug!("stream={} msg=shut down ack", config.name); + debug!("stream={} msg=shut down stream", config.name); } }); From 7f3b391c4e1df25f1878083ba9bcb7ec5ef74f34 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 26 Aug 2024 15:20:57 +0200 Subject: [PATCH 39/72] fix: bubble backfill transform. no op out of download metadata info. --- backfill/src/worker/transaction.rs | 61 ++++++++----------- core/src/metadata_json.rs | 14 ----- grpc-ingest/src/ingester.rs | 1 + grpc-ingest/src/redis.rs | 5 +- program_transformers/src/bubblegum/db.rs | 33 ++++++---- program_transformers/src/bubblegum/mint_v1.rs | 12 +--- .../src/bubblegum/transfer.rs | 1 + .../src/bubblegum/update_metadata.rs | 12 +--- 8 files changed, 59 insertions(+), 80 deletions(-) diff --git a/backfill/src/worker/transaction.rs b/backfill/src/worker/transaction.rs index 3bb34b4b0..6a25f6f1b 100644 --- a/backfill/src/worker/transaction.rs +++ b/backfill/src/worker/transaction.rs @@ -42,7 +42,6 @@ impl TryFrom for TransactionInfo { fn try_from( fetched_transaction: FetchedEncodedTransactionWithStatusMeta, ) -> Result { - tracing::info!("fetched transaction: {:?}", fetched_transaction); let mut account_keys = Vec::new(); let encoded_transaction_with_status_meta = fetched_transaction.0; @@ -62,49 +61,41 @@ impl TryFrom for TransactionInfo { .transaction .meta .ok_or(ErrorKind::Generic( - "unable to get meta from transaction".to_string(), + "transaction metadata is missing".to_string(), ))?; for address in msg.static_account_keys().iter().copied() { account_keys.push(address); } - let ui_loaded_addresses = meta.loaded_addresses; - let message_address_table_lookup = msg.address_table_lookups(); - - if message_address_table_lookup.is_some() { - if let OptionSerializer::Some(ui_lookup_table) = ui_loaded_addresses { - for address in ui_lookup_table.writable { - account_keys.push(PubkeyString(address).try_into()?); - } - - for address in ui_lookup_table.readonly { - account_keys.push(PubkeyString(address).try_into()?); - } + let ui_loaded_addresses = match meta.loaded_addresses { + OptionSerializer::Some(addresses) => addresses, + OptionSerializer::None => { + return Err(ErrorKind::Generic( + "loaded addresses data is missing".to_string(), + )) } - } - - let mut meta_inner_instructions = Vec::new(); + OptionSerializer::Skip => { + return Err(ErrorKind::Generic( + "loaded addresses are skipped".to_string(), + )); + } + }; - let compiled_instruction = msg.instructions().to_vec(); + let writtable_loaded_addresses = ui_loaded_addresses.writable; + let readable_loaded_addresses = ui_loaded_addresses.readonly; - let mut instructions = Vec::new(); + if msg.address_table_lookups().is_some() { + for address in writtable_loaded_addresses { + account_keys.push(PubkeyString(address).try_into()?); + } - for inner in compiled_instruction { - instructions.push(InnerInstruction { - stack_height: Some(0), - instruction: CompiledInstruction { - program_id_index: inner.program_id_index, - accounts: inner.accounts, - data: inner.data, - }, - }); + for address in readable_loaded_addresses { + account_keys.push(PubkeyString(address).try_into()?); + } } - meta_inner_instructions.push(InnerInstructions { - index: 0, - instructions, - }); + let mut meta_inner_instructions = Vec::new(); if let OptionSerializer::Some(inner_instructions) = meta.inner_instructions { for ix in inner_instructions { @@ -117,9 +108,9 @@ impl TryFrom for TransactionInfo { instruction: CompiledInstruction { program_id_index: compiled.program_id_index, accounts: compiled.accounts, - data: bs58::decode(compiled.data) - .into_vec() - .map_err(|e| ErrorKind::Generic(e.to_string()))?, + data: bs58::decode(compiled.data).into_vec().map_err(|e| { + ErrorKind::Generic(format!("Error decoding data: {}", e)) + })?, }, }); } diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs index b66c81cc3..36c3ebf0a 100644 --- a/core/src/metadata_json.rs +++ b/core/src/metadata_json.rs @@ -239,20 +239,6 @@ impl DownloadMetadata { &self, download_metadata_info: &DownloadMetadataInfo, ) -> Result<(), MetadataJsonTaskError> { - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(self.pool.clone()); - - if let Some(asset_data) = - asset_data::Entity::find_by_id(download_metadata_info.asset_data_id.clone()) - .one(&conn) - .await? - { - if asset_data.slot_updated == download_metadata_info.slot - && asset_data.reindex == Some(false) - { - return Ok(()); - } - } - perform_metadata_json_task( self.client.clone(), self.pool.clone(), diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 53f34ef1d..30a7924f6 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -150,6 +150,7 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { .connection(connection.clone()) .handler(move |info| { let pt_transactions = Arc::clone(&pt_transactions); + Box::pin(async move { let info = TransactionInfo::try_parse_msg(info)?; diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 1cd0d564b..feef7228c 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -558,11 +558,14 @@ impl IngestStream { break; }, claimed = self.pending(&mut connection, &start) => { - info!("redis=claimed stream={} claimed={:?}", config.name, claimed); if let Ok(Some(claimed)) = claimed { + let ids = claimed.ids.clone(); let ids: Vec<&str> = ids.iter().map(|info| info.id.as_str()).collect(); + info!("redis=claimed stream={} claimed={:?}", config.name, ids.len()); + + for StreamId { id, map } in claimed.ids.into_iter() { executor.push(IngestStreamJob::Process((id, map))); } diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index b086c6de8..2a3d2e2ad 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -1,5 +1,6 @@ use { crate::error::{ProgramTransformerError, ProgramTransformerResult}, + das_core::DownloadMetadataInfo, digital_asset_types::dao::{ asset, asset_authority, asset_creators, asset_data, asset_grouping, backfill_items, cl_audits_v2, cl_items, @@ -403,13 +404,11 @@ pub async fn upsert_asset_data( chain_data: JsonValue, metadata_url: String, metadata_mutability: Mutability, - metadata: JsonValue, slot_updated: i64, - reindex: Option, raw_name: Vec, raw_symbol: Vec, seq: i64, -) -> ProgramTransformerResult<()> +) -> ProgramTransformerResult> where T: ConnectionTrait + TransactionTrait, { @@ -417,11 +416,11 @@ where id: ActiveValue::Set(id.clone()), chain_data_mutability: ActiveValue::Set(chain_data_mutability), chain_data: ActiveValue::Set(chain_data), - metadata_url: ActiveValue::Set(metadata_url), + metadata_url: ActiveValue::Set(metadata_url.clone()), metadata_mutability: ActiveValue::Set(metadata_mutability), - metadata: ActiveValue::Set(metadata), + metadata: ActiveValue::Set(JsonValue::String("processing".to_string())), slot_updated: ActiveValue::Set(slot_updated), - reindex: ActiveValue::Set(reindex), + reindex: ActiveValue::Set(Some(true)), raw_name: ActiveValue::Set(Some(raw_name)), raw_symbol: ActiveValue::Set(Some(raw_symbol)), base_info_seq: ActiveValue::Set(Some(seq)), @@ -435,9 +434,7 @@ where asset_data::Column::ChainData, asset_data::Column::MetadataUrl, asset_data::Column::MetadataMutability, - // Don't update asset_data::Column::Metadata if it already exists. Even if we - // are indexing `update_metadata`` and there's a new URI, the new background - // task will overwrite it. + asset_data::Column::Metadata, asset_data::Column::SlotUpdated, asset_data::Column::Reindex, asset_data::Column::RawName, @@ -450,15 +447,27 @@ where // Do not overwrite changes that happened after decompression (asset_data.base_info_seq = 0). // Do not overwrite changes from a later Bubblegum instruction. + // Do not update the record if the incoming slot is larger than the current or if it's null. + // Update if the current slot on the record is null. query.sql = format!( - "{} WHERE (asset_data.base_info_seq != 0 AND excluded.base_info_seq >= asset_data.base_info_seq) OR asset_data.base_info_seq IS NULL", + "{} WHERE ((asset_data.base_info_seq != 0 AND excluded.base_info_seq >= asset_data.base_info_seq) OR asset_data.base_info_seq IS NULL) AND (excluded.slot_updated <= asset_data.slot_updated OR asset_data.slot_updated IS NULL)", query.sql ); - txn.execute(query) + + let result = txn + .execute(query) .await .map_err(|db_err| ProgramTransformerError::StorageWriteError(db_err.to_string()))?; - Ok(()) + if result.rows_affected() > 0 { + Ok(Some(DownloadMetadataInfo::new( + id, + metadata_url, + slot_updated, + ))) + } else { + Ok(None) + } } #[allow(clippy::too_many_arguments)] diff --git a/program_transformers/src/bubblegum/mint_v1.rs b/program_transformers/src/bubblegum/mint_v1.rs index bb2855b9a..47193ee1f 100644 --- a/program_transformers/src/bubblegum/mint_v1.rs +++ b/program_transformers/src/bubblegum/mint_v1.rs @@ -24,7 +24,7 @@ use { }, json::ChainDataV1, }, - sea_orm::{query::JsonValue, ConnectionTrait, TransactionTrait}, + sea_orm::{ConnectionTrait, TransactionTrait}, tracing::warn, }; @@ -91,16 +91,14 @@ where // automatically rolled back. let multi_txn = txn.begin().await?; - upsert_asset_data( + let download_metadata_info = upsert_asset_data( &multi_txn, id_bytes.to_vec(), chain_mutability, chain_data_json, uri.clone(), Mutability::Mutable, - JsonValue::String("processing".to_string()), slot_i, - Some(true), name.to_vec(), symbol.to_vec(), seq as i64, @@ -207,11 +205,7 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new( - id_bytes.to_vec(), - uri, - slot_i, - ))) + Ok(download_metadata_info) } _ => Err(ProgramTransformerError::NotImplemented), }; diff --git a/program_transformers/src/bubblegum/transfer.rs b/program_transformers/src/bubblegum/transfer.rs index 9c551beea..617efdf72 100644 --- a/program_transformers/src/bubblegum/transfer.rs +++ b/program_transformers/src/bubblegum/transfer.rs @@ -77,6 +77,7 @@ where } } } + Err(ProgramTransformerError::ParsingError( "Ix not parsed correctly".to_string(), )) diff --git a/program_transformers/src/bubblegum/update_metadata.rs b/program_transformers/src/bubblegum/update_metadata.rs index 2c55a3b8d..dff719fd4 100644 --- a/program_transformers/src/bubblegum/update_metadata.rs +++ b/program_transformers/src/bubblegum/update_metadata.rs @@ -22,7 +22,7 @@ use { }, json::ChainDataV1, }, - sea_orm::{query::*, ConnectionTrait, JsonValue}, + sea_orm::{query::*, ConnectionTrait}, tracing::warn, }; @@ -114,16 +114,14 @@ where // automatically rolled back. let multi_txn = txn.begin().await?; - upsert_asset_data( + let download_metadata_info = upsert_asset_data( &multi_txn, id_bytes.to_vec(), chain_mutability, chain_data_json, uri.clone(), Mutability::Mutable, - JsonValue::String("processing".to_string()), slot_i, - Some(true), name.into_bytes().to_vec(), symbol.into_bytes().to_vec(), seq as i64, @@ -188,11 +186,7 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new( - id_bytes.to_vec(), - uri, - slot_i, - ))) + Ok(download_metadata_info) } _ => Err(ProgramTransformerError::NotImplemented), }; From 4c3f6491d6e884d7c9fc57a8e6c8594b70258564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Rodr=C3=ADguez?= <_@kevinrodriguez.io> Date: Thu, 12 Sep 2024 21:28:31 -0600 Subject: [PATCH 40/72] Add force flag to TreeWorkerArgs for complete reindexing (#148) * feat: allows complete reindex * chore: allow deprecated borsh package --------- Co-authored-by: Kyle Espinola --- backfill/src/worker/tree.rs | 38 ++++++++++++++++++++++----------- blockbuster/src/lib.rs | 1 + tools/acc_forwarder/src/main.rs | 1 + 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/backfill/src/worker/tree.rs b/backfill/src/worker/tree.rs index e75917e52..e6db2b8d9 100644 --- a/backfill/src/worker/tree.rs +++ b/backfill/src/worker/tree.rs @@ -27,6 +27,9 @@ pub struct TreeWorkerArgs { #[clap(flatten)] pub program_transformer_worker: ProgramTransformerWorkerArgs, + + #[clap(long, env, default_value = "false")] + pub force: bool, } impl TreeWorkerArgs { pub fn start( @@ -44,6 +47,7 @@ impl TreeWorkerArgs { let program_transformer_worker_args = self.program_transformer_worker.clone(); let signature_worker_args = self.signature_worker.clone(); let gap_worker_args = self.gap_worker.clone(); + let force = self.force; tokio::spawn(async move { let (metadata_json_download_worker, metadata_json_download_sender) = @@ -67,23 +71,31 @@ impl TreeWorkerArgs { .map(TryInto::try_into) .collect::, _>>()?; - let upper_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_desc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; - - let lower_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_asc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; + let upper_known_seq = if force { + None + } else { + cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_desc(cl_audits_v2::Column::Seq) + .one(&conn) + .await? + }; + + let lower_known_seq = if force { + None + } else { + cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_asc(cl_audits_v2::Column::Seq) + .one(&conn) + .await? + }; if let Some(upper_seq) = upper_known_seq { let signature = Signature::try_from(upper_seq.tx.as_ref())?; - gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); - } else if tree.seq > 0 { + // Reprocess the entire tree if force is true or if the tree has a seq of 0 to keep the current behavior + } else if force || tree.seq > 0 { gaps.push(TreeGapFill::new(tree.pubkey, None, None)); } diff --git a/blockbuster/src/lib.rs b/blockbuster/src/lib.rs index 983c72897..e8e6eeedf 100644 --- a/blockbuster/src/lib.rs +++ b/blockbuster/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(deprecated)] pub mod error; pub mod instruction; pub mod program_handler; diff --git a/tools/acc_forwarder/src/main.rs b/tools/acc_forwarder/src/main.rs index 3c7c96279..c4345aaf2 100644 --- a/tools/acc_forwarder/src/main.rs +++ b/tools/acc_forwarder/src/main.rs @@ -1,3 +1,4 @@ +#![allow(deprecated)] use { anyhow::Context, clap::Parser, From 4e3c076f5733fb467b808581256aa6e6960a8e2d Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 16 Sep 2024 12:54:52 +0200 Subject: [PATCH 41/72] refactor: drop exculsion clauses on cl_items to allow for reindexing (#151) --- program_transformers/src/bubblegum/db.rs | 51 ++++-------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index 2a3d2e2ad..df7f12c15 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -2,8 +2,7 @@ use { crate::error::{ProgramTransformerError, ProgramTransformerResult}, das_core::DownloadMetadataInfo, digital_asset_types::dao::{ - asset, asset_authority, asset_creators, asset_data, asset_grouping, backfill_items, - cl_audits_v2, cl_items, + asset, asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, cl_items, sea_orm_active_enums::{ ChainMutability, Instruction, Mutability, OwnerType, RoyaltyTargetType, SpecificationAssetClass, SpecificationVersions, @@ -11,14 +10,14 @@ use { }, mpl_bubblegum::types::{Collection, Creator}, sea_orm::{ - entity::{ActiveValue, ColumnTrait, EntityTrait}, + entity::{ActiveValue, EntityTrait}, prelude::*, - query::{JsonValue, QueryFilter, QuerySelect, QueryTrait}, + query::{JsonValue, QueryTrait}, sea_query::query::OnConflict, ConnectionTrait, DbBackend, TransactionTrait, }, spl_account_compression::events::ChangeLogEventV1, - tracing::{debug, error, info}, + tracing::{debug, error}, }; pub async fn save_changelog_event<'c, T>( @@ -41,7 +40,7 @@ const fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { pub async fn insert_change_log<'c, T>( change_log_event: &ChangeLogEventV1, - slot: u64, + _slot: u64, txn_id: &str, txn: &T, instruction: &str, @@ -49,10 +48,9 @@ pub async fn insert_change_log<'c, T>( where T: ConnectionTrait + TransactionTrait, { - let mut i: i64 = 0; let depth = change_log_event.path.len() - 1; let tree_id = change_log_event.id.as_ref(); - for p in change_log_event.path.iter() { + for (i, p) in (0_i64..).zip(change_log_event.path.iter()) { let node_idx = p.index as i64; debug!( "seq {}, index {} level {}, node {:?}, txn: {:?}, instruction {}", @@ -79,8 +77,7 @@ where ..Default::default() }; - i += 1; - let mut query = cl_items::Entity::insert(item) + let query = cl_items::Entity::insert(item) .on_conflict( OnConflict::columns([cl_items::Column::Tree, cl_items::Column::NodeIdx]) .update_columns([ @@ -92,7 +89,6 @@ where .to_owned(), ) .build(DbBackend::Postgres); - query.sql = format!("{} WHERE excluded.seq > cl_items.seq", query.sql); txn.execute(query) .await .map_err(|db_err| ProgramTransformerError::StorageWriteError(db_err.to_string()))?; @@ -121,7 +117,7 @@ where cl_audits_v2::Column::LeafIdx, cl_audits_v2::Column::Seq, ]) - .do_nothing() + .update_columns([cl_audits_v2::Column::Tx, cl_audits_v2::Column::Instruction]) .to_owned(), ) .build(DbBackend::Postgres); @@ -132,37 +128,6 @@ where } } - // If and only if the entire path of nodes was inserted into the `cl_items` table, then insert - // a single row into the `backfill_items` table. This way if an incomplete path was inserted - // into `cl_items` due to an error, a gap will be created for the tree and the backfiller will - // fix it. - if i - 1 == depth as i64 { - // See if the tree already exists in the `backfill_items` table. - let rows = backfill_items::Entity::find() - .filter(backfill_items::Column::Tree.eq(tree_id)) - .limit(1) - .all(txn) - .await?; - - // If the tree does not exist in `backfill_items` and the sequence number is greater than 1, - // then we know we will need to backfill the tree from sequence number 1 up to the current - // sequence number. So in this case we set at flag to force checking the tree. - let force_chk = rows.is_empty() && change_log_event.seq > 1; - - info!("Adding to backfill_items table at level {}", i - 1); - let item = backfill_items::ActiveModel { - tree: ActiveValue::Set(tree_id.to_vec()), - seq: ActiveValue::Set(change_log_event.seq as i64), - slot: ActiveValue::Set(slot as i64), - force_chk: ActiveValue::Set(force_chk), - backfilled: ActiveValue::Set(false), - failed: ActiveValue::Set(false), - ..Default::default() - }; - - backfill_items::Entity::insert(item).exec(txn).await?; - } - Ok(()) } From c789b52eac4de4df8eebdf231c0ba53918642ea5 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 26 Sep 2024 12:46:31 +0200 Subject: [PATCH 42/72] feat: replay bubblegum transactions based on what is in cl_audits_v2 --- backfill/src/lib.rs | 108 ++++++++++++++++++++- backfill/src/worker/program_transformer.rs | 2 +- ops/src/bubblegum/cmd.rs | 7 +- ops/src/bubblegum/mod.rs | 1 + ops/src/bubblegum/replay.rs | 27 ++++++ 5 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 ops/src/bubblegum/replay.rs diff --git a/backfill/src/lib.rs b/backfill/src/lib.rs index c6973f85b..39a478170 100644 --- a/backfill/src/lib.rs +++ b/backfill/src/lib.rs @@ -3,14 +3,24 @@ mod gap; mod tree; mod worker; +use das_core::{MetadataJsonDownloadWorkerArgs, Rpc}; pub use error::ErrorKind; use anyhow::Result; use clap::Parser; -use das_core::Rpc; +use digital_asset_types::dao::cl_audits_v2; use futures::{stream::FuturesUnordered, StreamExt}; +use sea_orm::ColumnTrait; +use sea_orm::QueryOrder; +use sea_orm::SqlxPostgresConnector; +use sea_orm::{EntityTrait, QueryFilter}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Signature; +use std::str::FromStr; +use tracing::error; use tree::TreeResponse; -use worker::TreeWorkerArgs; +use worker::ProgramTransformerWorkerArgs; +use worker::{SignatureWorkerArgs, TreeWorkerArgs}; #[derive(Clone)] pub struct BubblegumBackfillContext { @@ -67,3 +77,97 @@ pub async fn start_bubblegum_backfill( Ok(()) } + +#[derive(Debug, Parser, Clone)] +pub struct BubblegumReplayArgs { + /// Number of tree crawler workers + #[arg(long, env, default_value = "20")] + pub tree_crawler_count: usize, + + /// The list of trees to crawl. If not specified, all trees will be crawled. + #[arg(long, env, use_value_delimiter = true)] + pub trees: Vec, + + #[clap(flatten)] + pub signature_worker: SignatureWorkerArgs, + + #[clap(flatten)] + pub program_transformer_worker: ProgramTransformerWorkerArgs, + + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, +} + +pub async fn start_bubblegum_replay( + context: BubblegumBackfillContext, + args: BubblegumReplayArgs, +) -> Result<()> { + let pubkeys = args + .trees + .iter() + .map(|tree| Pubkey::from_str(tree).map(|pubkey| pubkey.to_bytes().to_vec())) + .collect::>, _>>()?; + + let mut crawl_handles = FuturesUnordered::new(); + + for pubkey in pubkeys { + if crawl_handles.len() >= args.tree_crawler_count { + crawl_handles.next().await; + } + let database_pool = context.database_pool.clone(); + + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(database_pool); + + let cl_audits = cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(pubkey)) + .order_by_asc(cl_audits_v2::Column::Seq) + .all(&conn) + .await?; + + let context = context.clone(); + let metadata_json_download_worker_args = args.metadata_json_download_worker.clone(); + let program_transformer_worker_args = args.program_transformer_worker.clone(); + let signature_worker_args = args.signature_worker.clone(); + + let metadata_json_download_database_pool = context.database_pool.clone(); + let handle: tokio::task::JoinHandle> = tokio::spawn(async move { + let metadata_json_download_db_pool = metadata_json_download_database_pool.clone(); + let program_transformer_context = context.clone(); + let signature_context = context.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = + metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; + + let (program_transformer_worker, transaction_info_sender) = + program_transformer_worker_args + .start(program_transformer_context, metadata_json_download_sender)?; + + let (signature_worker, signature_sender) = + signature_worker_args.start(signature_context, transaction_info_sender)?; + + for audit in cl_audits { + let signature = Signature::try_from(audit.tx.as_ref())?; + if let Err(e) = signature_sender.send(signature).await { + error!("send signature: {:?}", e); + } + } + + drop(signature_sender); + + futures::future::try_join3( + signature_worker, + program_transformer_worker, + metadata_json_download_worker, + ) + .await?; + + Ok(()) + }); + + crawl_handles.push(handle); + } + + futures::future::try_join_all(crawl_handles).await?; + + Ok(()) +} diff --git a/backfill/src/worker/program_transformer.rs b/backfill/src/worker/program_transformer.rs index 1fa487b49..d0e365854 100644 --- a/backfill/src/worker/program_transformer.rs +++ b/backfill/src/worker/program_transformer.rs @@ -35,7 +35,7 @@ impl ProgramTransformerWorkerArgs { transactions.push(gap); } - transactions.sort_by(|a, b| b.signature.cmp(&a.signature)); + transactions.sort_by(|a, b| a.slot.cmp(&b.slot)); for transaction in transactions { if let Err(e) = program_transformer.handle_transaction(&transaction).await { diff --git a/ops/src/bubblegum/cmd.rs b/ops/src/bubblegum/cmd.rs index 1004a227d..517c80835 100644 --- a/ops/src/bubblegum/cmd.rs +++ b/ops/src/bubblegum/cmd.rs @@ -1,4 +1,4 @@ -use super::backfiller; +use super::{backfiller, replay}; use anyhow::Result; use clap::{Args, Subcommand}; @@ -8,6 +8,8 @@ pub enum Commands { /// It crawls through trees and backfills any missed tree transactions. #[clap(name = "backfill")] Backfill(backfiller::Args), + #[clap(name = "replay")] + Replay(replay::Args), } #[derive(Debug, Clone, Args)] @@ -21,6 +23,9 @@ pub async fn subcommand(subcommand: BubblegumCommand) -> Result<()> { Commands::Backfill(args) => { backfiller::run(args).await?; } + Commands::Replay(args) => { + replay::run(args).await?; + } } Ok(()) diff --git a/ops/src/bubblegum/mod.rs b/ops/src/bubblegum/mod.rs index d798b6e9e..6fd6d0efb 100644 --- a/ops/src/bubblegum/mod.rs +++ b/ops/src/bubblegum/mod.rs @@ -1,4 +1,5 @@ mod backfiller; mod cmd; +mod replay; pub use cmd::*; diff --git a/ops/src/bubblegum/replay.rs b/ops/src/bubblegum/replay.rs new file mode 100644 index 000000000..23647cbbe --- /dev/null +++ b/ops/src/bubblegum/replay.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use clap::Parser; +use das_backfill::{start_bubblegum_replay, BubblegumBackfillContext, BubblegumReplayArgs}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, + + #[clap(flatten)] + pub replay_bubblegum: BubblegumReplayArgs, +} + +pub async fn run(config: Args) -> Result<()> { + let database_pool = connect_db(&config.database).await?; + + let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumBackfillContext::new(database_pool, solana_rpc); + + start_bubblegum_replay(context, config.replay_bubblegum).await +} From 45a5dcfda5d191dc122c0ff9bcf0b6885429c303 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 2 Oct 2024 14:23:54 +0200 Subject: [PATCH 43/72] fix: order bubblegum backfill instructions by seq (#158) --- backfill/src/worker/program_transformer.rs | 138 +++++++++++++++++++-- program_transformers/src/lib.rs | 2 +- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/backfill/src/worker/program_transformer.rs b/backfill/src/worker/program_transformer.rs index d0e365854..5890e67d3 100644 --- a/backfill/src/worker/program_transformer.rs +++ b/backfill/src/worker/program_transformer.rs @@ -1,8 +1,15 @@ use anyhow::Result; +use blockbuster::{ + instruction::InstructionBundle, program_handler::ProgramParser, + programs::bubblegum::BubblegumParser, programs::ProgramParseResult, +}; use clap::Parser; use das_core::{create_download_metadata_notifier, DownloadMetadataInfo}; use log::error; -use program_transformers::{ProgramTransformer, TransactionInfo}; +use program_transformers::{ + bubblegum::handle_bubblegum_instruction, ProgramTransformer, TransactionInfo, +}; +use sea_orm::SqlxPostgresConnector; use tokio::sync::mpsc::{channel, Sender, UnboundedSender}; use tokio::task::JoinHandle; @@ -23,24 +30,133 @@ impl ProgramTransformerWorkerArgs { let (sender, mut receiver) = channel::(self.program_transformer_channel_size); + let worker_forwarder = forwarder.clone(); + let worker_pool = context.database_pool.clone(); let handle = tokio::spawn(async move { let mut transactions = Vec::new(); - let pool = context.database_pool.clone(); - let download_metadata_notifier = create_download_metadata_notifier(forwarder).await; + let download_metadata_notifier = + create_download_metadata_notifier(worker_forwarder.clone()).await; + let program_transformer = + ProgramTransformer::new(worker_pool.clone(), download_metadata_notifier); - let program_transformer = ProgramTransformer::new(pool, download_metadata_notifier); - - while let Some(gap) = receiver.recv().await { - transactions.push(gap); + while let Some(transaction) = receiver.recv().await { + transactions.push(transaction); } - transactions.sort_by(|a, b| a.slot.cmp(&b.slot)); + let mut instructions = transactions + .iter() + .flat_map(|tx_info| { + let ordered_instructions = program_transformer.break_transaction(tx_info); + ordered_instructions.into_iter().map(|(ix_pair, inner_ix)| { + ( + tx_info.signature.to_string(), + ix_pair.0, + ix_pair.1, + inner_ix, + ix_pair + .1 + .accounts + .iter() + .map(|&i| tx_info.account_keys[i as usize]) + .collect::>(), + tx_info.slot, + ) + }) + }) + .collect::>(); + instructions.sort_by(|a, b| { + let a_tree_update_seq = if let Some(program_parser) = + program_transformer.match_program(&a.1) + { + if let Ok(result) = program_parser.handle_instruction(&InstructionBundle { + txn_id: &a.0, + program: a.1, + instruction: Some(a.2), + inner_ix: a.3.as_deref(), + keys: a.4.as_slice(), + slot: a.5, + }) { + if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() + { + parsing_result + .tree_update + .as_ref() + .map_or(u64::MAX, |event| event.seq) + } else { + u64::MAX + } + } else { + u64::MAX + } + } else { + u64::MAX + }; + + let b_tree_update_seq = if let Some(program_parser) = + program_transformer.match_program(&b.1) + { + if let Ok(result) = program_parser.handle_instruction(&InstructionBundle { + txn_id: &b.0, + program: b.1, + instruction: Some(b.2), + inner_ix: b.3.as_deref(), + keys: b.4.as_slice(), + slot: b.5, + }) { + if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() + { + parsing_result + .tree_update + .as_ref() + .map_or(u64::MAX, |event| event.seq) + } else { + u64::MAX + } + } else { + u64::MAX + } + } else { + u64::MAX + }; + + a_tree_update_seq.cmp(&b_tree_update_seq) + }); + + let parser = BubblegumParser {}; - for transaction in transactions { - if let Err(e) = program_transformer.handle_transaction(&transaction).await { - error!("handle transaction: {:?}", e) + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(worker_pool); + + for i in instructions { + let bundle = &InstructionBundle { + txn_id: &i.0, + program: i.1, + instruction: Some(i.2), + inner_ix: i.3.as_deref(), + keys: i.4.as_slice(), + slot: i.5, }; + if let Ok(result) = parser.handle_instruction(bundle) { + if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() { + let download_metadata_notifier = + create_download_metadata_notifier(worker_forwarder.clone()).await; + + if let Err(err) = handle_bubblegum_instruction( + parsing_result, + bundle, + &conn, + &download_metadata_notifier, + ) + .await + { + error!( + "Failed to handle bubblegum instruction for txn {:?}: {:?}", + bundle.txn_id, err + ); + break; + } + } + } } }); diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index de4eaca6d..c34fc82cf 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -30,7 +30,7 @@ use { }; mod asset_upserts; -mod bubblegum; +pub mod bubblegum; pub mod error; mod mpl_core_program; mod token; From f4744c0ce601507d5dd78a78395ec5b428c489af Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 7 Oct 2024 18:53:27 +0200 Subject: [PATCH 44/72] include back seq checks --- program_transformers/src/bubblegum/db.rs | 5 +++-- program_transformers/src/bubblegum/mod.rs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index df7f12c15..4c8c33f4a 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -77,7 +77,7 @@ where ..Default::default() }; - let query = cl_items::Entity::insert(item) + let mut query = cl_items::Entity::insert(item) .on_conflict( OnConflict::columns([cl_items::Column::Tree, cl_items::Column::NodeIdx]) .update_columns([ @@ -89,6 +89,7 @@ where .to_owned(), ) .build(DbBackend::Postgres); + query.sql = format!("{} WHERE excluded.seq > cl_items.seq", query.sql); txn.execute(query) .await .map_err(|db_err| ProgramTransformerError::StorageWriteError(db_err.to_string()))?; @@ -117,7 +118,7 @@ where cl_audits_v2::Column::LeafIdx, cl_audits_v2::Column::Seq, ]) - .update_columns([cl_audits_v2::Column::Tx, cl_audits_v2::Column::Instruction]) + .do_nothing() .to_owned(), ) .build(DbBackend::Postgres); diff --git a/program_transformers/src/bubblegum/mod.rs b/program_transformers/src/bubblegum/mod.rs index 5cdff2b8a..03a800858 100644 --- a/program_transformers/src/bubblegum/mod.rs +++ b/program_transformers/src/bubblegum/mod.rs @@ -11,7 +11,7 @@ use { token_metadata::types::UseMethod as TokenMetadataUseMethod, }, sea_orm::{ConnectionTrait, TransactionTrait}, - tracing::{debug, info}, + tracing::debug, }; mod burn; @@ -58,7 +58,7 @@ where InstructionName::SetDecompressibleState => "SetDecompressibleState", InstructionName::UpdateMetadata => "UpdateMetadata", }; - info!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id); + debug!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id); match ix_type { InstructionName::Transfer => { From 1afee4559c6f481995b8d9ccef85e2ab7b27a0b7 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 7 Oct 2024 20:52:33 +0200 Subject: [PATCH 45/72] Remove enforcing seq in bubblegum backfill --- backfill/src/worker/program_transformer.rs | 155 ++++----------------- 1 file changed, 30 insertions(+), 125 deletions(-) diff --git a/backfill/src/worker/program_transformer.rs b/backfill/src/worker/program_transformer.rs index 5890e67d3..6e7c20646 100644 --- a/backfill/src/worker/program_transformer.rs +++ b/backfill/src/worker/program_transformer.rs @@ -1,15 +1,11 @@ use anyhow::Result; -use blockbuster::{ - instruction::InstructionBundle, program_handler::ProgramParser, - programs::bubblegum::BubblegumParser, programs::ProgramParseResult, -}; use clap::Parser; use das_core::{create_download_metadata_notifier, DownloadMetadataInfo}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use log::error; -use program_transformers::{ - bubblegum::handle_bubblegum_instruction, ProgramTransformer, TransactionInfo, -}; -use sea_orm::SqlxPostgresConnector; +use program_transformers::{ProgramTransformer, TransactionInfo}; +use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender, UnboundedSender}; use tokio::task::JoinHandle; @@ -19,6 +15,8 @@ use crate::BubblegumBackfillContext; pub struct ProgramTransformerWorkerArgs { #[arg(long, env, default_value = "100000")] pub program_transformer_channel_size: usize, + #[arg(long, env, default_value = "50")] + pub program_transformer_worker_count: usize, } impl ProgramTransformerWorkerArgs { @@ -32,132 +30,39 @@ impl ProgramTransformerWorkerArgs { let worker_forwarder = forwarder.clone(); let worker_pool = context.database_pool.clone(); + let worker_count = self.program_transformer_worker_count; let handle = tokio::spawn(async move { - let mut transactions = Vec::new(); - let download_metadata_notifier = create_download_metadata_notifier(worker_forwarder.clone()).await; - let program_transformer = - ProgramTransformer::new(worker_pool.clone(), download_metadata_notifier); - - while let Some(transaction) = receiver.recv().await { - transactions.push(transaction); - } - - let mut instructions = transactions - .iter() - .flat_map(|tx_info| { - let ordered_instructions = program_transformer.break_transaction(tx_info); - ordered_instructions.into_iter().map(|(ix_pair, inner_ix)| { - ( - tx_info.signature.to_string(), - ix_pair.0, - ix_pair.1, - inner_ix, - ix_pair - .1 - .accounts - .iter() - .map(|&i| tx_info.account_keys[i as usize]) - .collect::>(), - tx_info.slot, - ) - }) - }) - .collect::>(); - instructions.sort_by(|a, b| { - let a_tree_update_seq = if let Some(program_parser) = - program_transformer.match_program(&a.1) - { - if let Ok(result) = program_parser.handle_instruction(&InstructionBundle { - txn_id: &a.0, - program: a.1, - instruction: Some(a.2), - inner_ix: a.3.as_deref(), - keys: a.4.as_slice(), - slot: a.5, - }) { - if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() - { - parsing_result - .tree_update - .as_ref() - .map_or(u64::MAX, |event| event.seq) - } else { - u64::MAX - } - } else { - u64::MAX - } - } else { - u64::MAX - }; - - let b_tree_update_seq = if let Some(program_parser) = - program_transformer.match_program(&b.1) - { - if let Ok(result) = program_parser.handle_instruction(&InstructionBundle { - txn_id: &b.0, - program: b.1, - instruction: Some(b.2), - inner_ix: b.3.as_deref(), - keys: b.4.as_slice(), - slot: b.5, - }) { - if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() - { - parsing_result - .tree_update - .as_ref() - .map_or(u64::MAX, |event| event.seq) - } else { - u64::MAX - } - } else { - u64::MAX - } - } else { - u64::MAX - }; + let program_transformer = Arc::new(ProgramTransformer::new( + worker_pool.clone(), + download_metadata_notifier, + )); - a_tree_update_seq.cmp(&b_tree_update_seq) - }); + let mut handlers = FuturesUnordered::new(); - let parser = BubblegumParser {}; - - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(worker_pool); - - for i in instructions { - let bundle = &InstructionBundle { - txn_id: &i.0, - program: i.1, - instruction: Some(i.2), - inner_ix: i.3.as_deref(), - keys: i.4.as_slice(), - slot: i.5, - }; - if let Ok(result) = parser.handle_instruction(bundle) { - if let ProgramParseResult::Bubblegum(parsing_result) = result.result_type() { - let download_metadata_notifier = - create_download_metadata_notifier(worker_forwarder.clone()).await; + while let Some(transaction) = receiver.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } - if let Err(err) = handle_bubblegum_instruction( - parsing_result, - bundle, - &conn, - &download_metadata_notifier, - ) + let program_transformer_clone = Arc::clone(&program_transformer); + let handle = tokio::spawn(async move { + if let Err(err) = program_transformer_clone + .handle_transaction(&transaction) .await - { - error!( - "Failed to handle bubblegum instruction for txn {:?}: {:?}", - bundle.txn_id, err - ); - break; - } + { + error!( + "Failed to handle bubblegum instruction for txn {:?}: {:?}", + transaction.signature, err + ); } - } + }); + + handlers.push(handle); } + + futures::future::join_all(handlers).await; }); Ok((handle, sender)) From 18411a913fc00679698511bd56d58f8407d885bf Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Tue, 8 Oct 2024 14:08:39 +0200 Subject: [PATCH 46/72] Replay single tree with targeted seq replay (#159) --- backfill/src/lib.rs | 101 ++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 59 deletions(-) diff --git a/backfill/src/lib.rs b/backfill/src/lib.rs index 39a478170..dac3aba44 100644 --- a/backfill/src/lib.rs +++ b/backfill/src/lib.rs @@ -80,13 +80,13 @@ pub async fn start_bubblegum_backfill( #[derive(Debug, Parser, Clone)] pub struct BubblegumReplayArgs { - /// Number of tree crawler workers - #[arg(long, env, default_value = "20")] - pub tree_crawler_count: usize, + /// The tree to replay. + #[arg(long, env)] + pub tree: String, - /// The list of trees to crawl. If not specified, all trees will be crawled. + /// The list of sequences to replay. If not specified, all sequences will be replayed. #[arg(long, env, use_value_delimiter = true)] - pub trees: Vec, + pub only_sequences: Option>, #[clap(flatten)] pub signature_worker: SignatureWorkerArgs, @@ -102,72 +102,55 @@ pub async fn start_bubblegum_replay( context: BubblegumBackfillContext, args: BubblegumReplayArgs, ) -> Result<()> { - let pubkeys = args - .trees - .iter() - .map(|tree| Pubkey::from_str(tree).map(|pubkey| pubkey.to_bytes().to_vec())) - .collect::>, _>>()?; - - let mut crawl_handles = FuturesUnordered::new(); - - for pubkey in pubkeys { - if crawl_handles.len() >= args.tree_crawler_count { - crawl_handles.next().await; - } - let database_pool = context.database_pool.clone(); - - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(database_pool); + let pubkey = Pubkey::from_str(&args.tree) + .map(|pubkey| pubkey.to_bytes().to_vec()) + .map_err(|e| anyhow::anyhow!("Invalid tree pubkey: {:?}", e))?; - let cl_audits = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(pubkey)) - .order_by_asc(cl_audits_v2::Column::Seq) - .all(&conn) - .await?; + let database_pool = context.database_pool.clone(); + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(database_pool); - let context = context.clone(); - let metadata_json_download_worker_args = args.metadata_json_download_worker.clone(); - let program_transformer_worker_args = args.program_transformer_worker.clone(); - let signature_worker_args = args.signature_worker.clone(); + let mut query = cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(pubkey)) + .order_by_asc(cl_audits_v2::Column::Seq); - let metadata_json_download_database_pool = context.database_pool.clone(); - let handle: tokio::task::JoinHandle> = tokio::spawn(async move { - let metadata_json_download_db_pool = metadata_json_download_database_pool.clone(); - let program_transformer_context = context.clone(); - let signature_context = context.clone(); - - let (metadata_json_download_worker, metadata_json_download_sender) = - metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; + if let Some(sequences) = args.only_sequences { + query = query.filter(cl_audits_v2::Column::Seq.is_in(sequences)); + } - let (program_transformer_worker, transaction_info_sender) = - program_transformer_worker_args - .start(program_transformer_context, metadata_json_download_sender)?; + let cl_audits = query.all(&conn).await?; - let (signature_worker, signature_sender) = - signature_worker_args.start(signature_context, transaction_info_sender)?; + let metadata_json_download_worker_args = args.metadata_json_download_worker.clone(); + let program_transformer_worker_args = args.program_transformer_worker.clone(); + let signature_worker_args = args.signature_worker.clone(); - for audit in cl_audits { - let signature = Signature::try_from(audit.tx.as_ref())?; - if let Err(e) = signature_sender.send(signature).await { - error!("send signature: {:?}", e); - } - } + let metadata_json_download_db_pool = context.database_pool.clone(); + let program_transformer_context = context.clone(); + let signature_context = context.clone(); - drop(signature_sender); + let (metadata_json_download_worker, metadata_json_download_sender) = + metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; - futures::future::try_join3( - signature_worker, - program_transformer_worker, - metadata_json_download_worker, - ) - .await?; + let (program_transformer_worker, transaction_info_sender) = program_transformer_worker_args + .start(program_transformer_context, metadata_json_download_sender)?; - Ok(()) - }); + let (signature_worker, signature_sender) = + signature_worker_args.start(signature_context, transaction_info_sender)?; - crawl_handles.push(handle); + for audit in cl_audits { + let signature = Signature::try_from(audit.tx.as_ref())?; + if let Err(e) = signature_sender.send(signature).await { + error!("send signature: {:?}", e); + } } - futures::future::try_join_all(crawl_handles).await?; + drop(signature_sender); + + futures::future::try_join3( + signature_worker, + program_transformer_worker, + metadata_json_download_worker, + ) + .await?; Ok(()) } From 110f71c5abb148394ccb823354a85d053d923a0e Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 9 Oct 2024 14:59:28 +0200 Subject: [PATCH 47/72] Support Account Snapshot Flushes (#154) * feat: mock snapshots code * refactor: remove dead code * feat: getting snapshot stream in the works * feat: adds snapshot stream * feat: adds config-ingester settings * Remove dead config * Add snapshot config in ingester --------- Co-authored-by: Kevin Rodriguez <_@kevinrodriguez.io> --- grpc-ingest/config-download-metadata.yml | 12 - grpc-ingest/config-ingester.yml | 30 +- grpc-ingest/src/config.rs | 268 +----------- grpc-ingest/src/download_metadata.rs | 236 ----------- grpc-ingest/src/grpc.rs | 205 +--------- grpc-ingest/src/ingester.rs | 334 +++------------ grpc-ingest/src/main.rs | 19 +- grpc-ingest/src/postgres.rs | 8 - grpc-ingest/src/prom.rs | 8 - grpc-ingest/src/redis.rs | 497 +---------------------- 10 files changed, 76 insertions(+), 1541 deletions(-) delete mode 100644 grpc-ingest/config-download-metadata.yml delete mode 100644 grpc-ingest/src/download_metadata.rs diff --git a/grpc-ingest/config-download-metadata.yml b/grpc-ingest/config-download-metadata.yml deleted file mode 100644 index c42307be4..000000000 --- a/grpc-ingest/config-download-metadata.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Important: only ONE `download-metadata` instance is supported right now! -prometheus: 127.0.0.1:8875 -postgres: - url: postgres://solana:solana@localhost/solana - min_connections: 10 - max_connections: 50 -download_metadata: - max_in_process: 50 # maximum tasks in process (downloading metadata) - prefetch_queue_size: 100 - limit_to_fetch: 200 # maximum number of tasks fetched from database - wait_tasks_max_idle_ms: 100 # if we do not have pending tasks, wait max ms - download_timeout_ms: 5_000 diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 38404071b..02c6de877 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -1,29 +1,15 @@ prometheus: 127.0.0.1:8875 -redis: - url: redis://localhost:6379 - group: ingester - consumer: consumer # every new ingester instance should have uniq name - - streams: - - type: account - stream: ACCOUNTS - - type: transaction - stream: TRANSACTIONS - xack_batch_max_size: 100 - xack_batch_max_idle_ms: 10 - xack_max_in_process: 100 - - type: metadatajson - stream: METADATA_JSON - prefetch_queue_size: 5_000 # max number of messages available in the read queue for processing - xpending_max: 1_000 # used for reading pending messages - xpending_only: false # exit once all pending messages consumed (should be applied if you want downscale number of ingesters) - xreadgroup_max: 1_000 # applied per each stream in one request +redis: redis://localhost:6379 postgres: url: postgres://solana:solana@localhost/solana min_connections: 10 max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible -program_transformer: - max_tasks_in_process: 100 +snapshots: + name: SNAPSHOTS + max_concurrency: 10 + batch_size: 100 + xack_batch_max_idle_ms: 1_000 + xack_buffer_size: 10_000 accounts: name: ACCOUNTS max_concurrency: 10 @@ -34,5 +20,5 @@ transactions: name: TRANSACTIONS download_metadata: max_attempts: 3 - stream_config: + stream: name: METADATA_JSON diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index ebd47ea5b..5d998bb4f 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -9,9 +9,6 @@ use { }, }; -pub const REDIS_STREAM_ACCOUNTS: &str = "ACCOUNTS"; -pub const REDIS_STREAM_TRANSACTIONS: &str = "TRANSACTIONS"; -pub const REDIS_STREAM_METADATA_JSON: &str = "METADATA_JSON"; pub const REDIS_STREAM_DATA_KEY: &str = "data"; pub async fn load(path: impl AsRef + Copy) -> anyhow::Result @@ -124,21 +121,6 @@ pub struct ConfigGrpc { pub geyser_endpoint: String, - #[serde(default = "ConfigGrpc::default_request_snapshot")] - pub request_snapshot: bool, - - #[serde( - default = "ConfigGrpc::default_geyser_update_message_buffer_size", - deserialize_with = "deserialize_usize_str" - )] - pub geyser_update_message_buffer_size: usize, - - #[serde( - default = "ConfigGrpc::solana_seen_event_cache_max_size", - deserialize_with = "deserialize_usize_str" - )] - pub solana_seen_event_cache_max_size: usize, - pub redis: ConfigGrpcRedis, #[serde( @@ -149,26 +131,13 @@ pub struct ConfigGrpc { } impl ConfigGrpc { - pub const fn default_request_snapshot() -> bool { - false - } - pub const fn default_max_concurrency() -> usize { 10 } - - pub const fn default_geyser_update_message_buffer_size() -> usize { - 100_000 - } - - pub const fn solana_seen_event_cache_max_size() -> usize { - 10_000 - } } #[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpcAccounts { - #[serde(default = "ConfigGrpcAccounts::default_stream")] pub stream: String, #[serde( default = "ConfigGrpcAccounts::default_stream_maxlen", @@ -182,10 +151,6 @@ pub struct ConfigGrpcAccounts { } impl ConfigGrpcAccounts { - pub fn default_stream() -> String { - REDIS_STREAM_ACCOUNTS.to_owned() - } - pub const fn default_stream_maxlen() -> usize { 100_000_000 } @@ -258,137 +223,21 @@ where #[derive(Debug, Clone, Deserialize)] pub struct ConfigIngester { - pub redis: ConfigIngesterRedis, + pub redis: String, pub postgres: ConfigIngesterPostgres, pub download_metadata: ConfigIngesterDownloadMetadata, - pub program_transformer: ConfigIngesterProgramTransformer, + pub snapshots: ConfigIngestStream, pub accounts: ConfigIngestStream, pub transactions: ConfigIngestStream, } -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigIngesterRedis { - pub url: String, - #[serde(default = "ConfigIngesterRedis::default_group")] - pub group: String, - #[serde(default = "ConfigIngesterRedis::default_consumer")] - pub consumer: String, - pub streams: Vec, - #[serde( - default = "ConfigIngesterRedis::default_prefetch_queue_size", - deserialize_with = "deserialize_usize_str" - )] - pub prefetch_queue_size: usize, - #[serde( - default = "ConfigIngesterRedis::default_xpending_max", - deserialize_with = "deserialize_usize_str" - )] - pub xpending_max: usize, - #[serde(default = "ConfigIngesterRedis::default_xpending_only")] - pub xpending_only: bool, - #[serde( - default = "ConfigIngesterRedis::default_xreadgroup_max", - deserialize_with = "deserialize_usize_str" - )] - pub xreadgroup_max: usize, -} - -impl ConfigIngesterRedis { - pub fn default_group() -> String { - "ingester".to_owned() - } - - pub fn default_consumer() -> String { - "consumer".to_owned() - } - - pub const fn default_prefetch_queue_size() -> usize { - 1_000 - } - - pub const fn default_xpending_max() -> usize { - 100 - } - - pub const fn default_xpending_only() -> bool { - false - } - - pub const fn default_xreadgroup_max() -> usize { - 1_000 - } -} - -#[derive(Debug, Clone, Copy)] -pub struct ConfigIngesterRedisStream { - pub stream_type: ConfigIngesterRedisStreamType, - pub stream: &'static str, - pub xack_batch_max_size: usize, - pub xack_batch_max_idle: Duration, - pub xack_max_in_process: usize, -} - -impl<'de> Deserialize<'de> for ConfigIngesterRedisStream { - fn deserialize(deserializer: D) -> Result - where - D: de::Deserializer<'de>, - { - #[derive(Debug, Clone, Copy, Deserialize)] - struct Raw { - #[serde(rename = "type")] - pub stream_type: ConfigIngesterRedisStreamType, - #[serde( - default = "default_xack_batch_max_size", - deserialize_with = "deserialize_usize_str" - )] - pub xack_batch_max_size: usize, - #[serde( - default = "default_xack_batch_max_idle", - deserialize_with = "deserialize_duration_str", - rename = "xack_batch_max_idle_ms" - )] - pub xack_batch_max_idle: Duration, - #[serde( - default = "default_xack_max_in_process", - deserialize_with = "deserialize_usize_str" - )] - pub xack_max_in_process: usize, - } - - const fn default_xack_batch_max_size() -> usize { - 100 - } - - const fn default_xack_batch_max_idle() -> Duration { - Duration::from_millis(10) - } - - const fn default_xack_max_in_process() -> usize { - 100 - } - - let raw = Raw::deserialize(deserializer)?; - - Ok(Self { - stream_type: raw.stream_type, - stream: match raw.stream_type { - ConfigIngesterRedisStreamType::Account => REDIS_STREAM_ACCOUNTS, - ConfigIngesterRedisStreamType::Transaction => REDIS_STREAM_TRANSACTIONS, - ConfigIngesterRedisStreamType::MetadataJson => REDIS_STREAM_METADATA_JSON, - }, - xack_batch_max_size: raw.xack_batch_max_size, - xack_batch_max_idle: raw.xack_batch_max_idle, - xack_max_in_process: raw.xack_max_in_process, - }) - } -} - #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ConfigIngesterRedisStreamType { Account, Transaction, MetadataJson, + Snapshot, } #[derive(Debug, Clone, Deserialize)] @@ -416,51 +265,9 @@ impl ConfigIngesterPostgres { } } -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigIngesterProgramTransformer { - #[serde( - default = "ConfigIngesterProgramTransformer::default_max_tasks_in_process", - deserialize_with = "deserialize_usize_str" - )] - pub max_tasks_in_process: usize, - #[serde( - default = "ConfigIngesterProgramTransformer::default_account_num_threads", - deserialize_with = "deserialize_usize_str" - )] - pub account_num_threads: usize, - #[serde( - default = "ConfigIngesterProgramTransformer::default_transaction_num_threads", - deserialize_with = "deserialize_usize_str" - )] - pub transaction_num_threads: usize, - #[serde( - default = "ConfigIngesterProgramTransformer::default_metadata_json_num_threads", - deserialize_with = "deserialize_usize_str" - )] - pub metadata_json_num_threads: usize, -} - -impl ConfigIngesterProgramTransformer { - pub const fn default_account_num_threads() -> usize { - 5 - } - - pub const fn default_transaction_num_threads() -> usize { - 1 - } - - pub const fn default_metadata_json_num_threads() -> usize { - 1 - } - - pub const fn default_max_tasks_in_process() -> usize { - 40 - } -} - #[derive(Debug, Clone, Default, Deserialize)] pub struct ConfigIngesterDownloadMetadata { - pub stream_config: ConfigIngestStream, + pub stream: ConfigIngestStream, #[serde( default = "ConfigIngesterDownloadMetadata::default_num_threads", deserialize_with = "deserialize_usize_str" @@ -482,8 +289,6 @@ pub struct ConfigIngesterDownloadMetadata { deserialize_with = "deserialize_usize_str" )] pub stream_maxlen: usize, - #[serde(default = "ConfigIngesterDownloadMetadata::default_stream")] - pub stream: String, #[serde( default = "ConfigIngesterDownloadMetadata::default_stream_max_size", deserialize_with = "deserialize_usize_str" @@ -510,10 +315,6 @@ impl ConfigIngesterDownloadMetadata { 10 } - pub fn default_stream() -> String { - REDIS_STREAM_METADATA_JSON.to_owned() - } - pub const fn default_stream_maxlen() -> usize { 10_000_000 } @@ -526,64 +327,3 @@ impl ConfigIngesterDownloadMetadata { Duration::from_millis(3_000) } } - -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigDownloadMetadata { - pub postgres: ConfigIngesterPostgres, - pub download_metadata: ConfigDownloadMetadataOpts, -} - -#[derive(Debug, Clone, Copy, Deserialize)] -pub struct ConfigDownloadMetadataOpts { - #[serde( - default = "ConfigDownloadMetadataOpts::default_max_in_process", - deserialize_with = "deserialize_usize_str" - )] - pub max_in_process: usize, - #[serde( - default = "ConfigDownloadMetadataOpts::default_prefetch_queue_size", - deserialize_with = "deserialize_usize_str" - )] - pub prefetch_queue_size: usize, - #[serde( - default = "ConfigDownloadMetadataOpts::default_limit_to_fetch", - deserialize_with = "deserialize_usize_str" - )] - pub limit_to_fetch: usize, - #[serde( - default = "ConfigDownloadMetadataOpts::default_wait_tasks_max_idle", - deserialize_with = "deserialize_duration_str", - rename = "wait_tasks_max_idle_ms" - )] - pub wait_tasks_max_idle: Duration, - #[serde( - default = "ConfigDownloadMetadataOpts::default_download_timeout", - deserialize_with = "deserialize_duration_str", - rename = "download_timeout_ms" - )] - pub download_timeout: Duration, - - pub stream: ConfigIngesterRedisStream, -} - -impl ConfigDownloadMetadataOpts { - pub const fn default_max_in_process() -> usize { - 50 - } - - pub const fn default_prefetch_queue_size() -> usize { - 100 - } - - pub const fn default_limit_to_fetch() -> usize { - 200 - } - - pub const fn default_wait_tasks_max_idle() -> Duration { - Duration::from_millis(100) - } - - pub const fn default_download_timeout() -> Duration { - Duration::from_millis(5_000) - } -} diff --git a/grpc-ingest/src/download_metadata.rs b/grpc-ingest/src/download_metadata.rs deleted file mode 100644 index 1e8e8a354..000000000 --- a/grpc-ingest/src/download_metadata.rs +++ /dev/null @@ -1,236 +0,0 @@ -use { - crate::{ - config::{ConfigDownloadMetadata, ConfigDownloadMetadataOpts}, - postgres::{create_pool as pg_create_pool, metrics_pgpool}, - util::create_shutdown, - }, - das_core::DownloadMetadataInfo, - digital_asset_types::dao::{asset_data, sea_orm_active_enums::TaskStatus, tasks}, - futures::{ - future::{pending, FutureExt}, - stream::StreamExt, - }, - reqwest::{ClientBuilder, StatusCode}, - sea_orm::{ - entity::{ActiveValue, ColumnTrait, EntityTrait}, - query::{Condition, Order, QueryFilter, QueryOrder, QuerySelect}, - sea_query::expr::Expr, - SqlxPostgresConnector, TransactionTrait, - }, - sqlx::PgPool, - std::{sync::Arc, time::Duration}, - tokio::{ - sync::{mpsc, Notify}, - task::JoinSet, - time::sleep, - }, - tracing::{info, warn}, -}; - -pub const TASK_TYPE: &str = "DownloadMetadata"; - -pub async fn run(config: ConfigDownloadMetadata) -> anyhow::Result<()> { - let mut shutdown = create_shutdown()?; - - // open connection to postgres - let pool = pg_create_pool(config.postgres).await?; - tokio::spawn({ - let pool = pool.clone(); - async move { metrics_pgpool(pool).await } - }); - - // reset previously runned tasks - tokio::select! { - result = reset_pending_tasks(pool.clone()) => { - let updated = result?; - info!("Reset {updated} tasks to Pending status"); - }, - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - return Ok(()) - }, - } - - // prefetch queue - let (tasks_tx, mut tasks_rx) = mpsc::channel(config.download_metadata.prefetch_queue_size); - let prefetch_shutdown = Arc::new(Notify::new()); - let prefetch_jh = { - let pool = pool.clone(); - let download_metadata = config.download_metadata; - let shutdown = Arc::clone(&prefetch_shutdown); - async move { - tokio::select! { - result = get_pending_tasks(pool, tasks_tx, download_metadata) => result, - _ = shutdown.notified() => Ok(()) - } - } - }; - tokio::pin!(prefetch_jh); - - // process tasks - let mut tasks = JoinSet::new(); - loop { - let pending_task_fut = if tasks.len() >= config.download_metadata.max_in_process { - pending().boxed() - } else { - tasks_rx.recv().boxed() - }; - - let tasks_fut = if tasks.is_empty() { - pending().boxed() - } else { - tasks.join_next().boxed() - }; - - tokio::select! { - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - break Ok(()); - }, - result = &mut prefetch_jh => break result, - Some(result) = tasks_fut => { - result??; - }, - Some(pending_task) = pending_task_fut => { - tasks.spawn(execute_task(pool.clone(), pending_task, config.download_metadata.download_timeout)); - } - }; - }?; - - tokio::select! { - Some(signal) = shutdown.next() => { - anyhow::bail!("{signal} received, force shutdown..."); - } - result = async move { - // shutdown `prefetch` channel - prefetch_shutdown.notify_one(); - // wait all spawned tasks - while let Some(result) = tasks.join_next().await { - result??; - } - // shutdown database connection - pool.close().await; - Ok::<(), anyhow::Error>(()) - } => result, - } -} - -// On startup reset tasks status -async fn reset_pending_tasks(pool: PgPool) -> anyhow::Result { - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - tasks::Entity::update_many() - .set(tasks::ActiveModel { - status: ActiveValue::Set(TaskStatus::Pending), - ..Default::default() - }) - .filter( - Condition::all() - .add(tasks::Column::Status.eq(TaskStatus::Running)) - .add(tasks::Column::TaskType.eq(TASK_TYPE)), - ) - .exec(&conn) - .await - .map(|result| result.rows_affected) - .map_err(Into::into) -} - -// Select Pending tasks, update status to Running and send to prefetch queue -async fn get_pending_tasks( - pool: PgPool, - tasks_tx: mpsc::Sender, - config: ConfigDownloadMetadataOpts, -) -> anyhow::Result<()> { - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - loop { - let pending_tasks = tasks::Entity::find() - .filter( - Condition::all() - .add(tasks::Column::Status.eq(TaskStatus::Pending)) - .add( - Expr::col(tasks::Column::Attempts) - .less_or_equal(Expr::col(tasks::Column::MaxAttempts)), - ), - ) - .order_by(tasks::Column::Attempts, Order::Asc) - .order_by(tasks::Column::CreatedAt, Order::Desc) - .limit(config.limit_to_fetch as u64) - .all(&conn) - .await?; - - if pending_tasks.is_empty() { - sleep(config.wait_tasks_max_idle).await; - } else { - tasks::Entity::update_many() - .set(tasks::ActiveModel { - status: ActiveValue::Set(TaskStatus::Running), - ..Default::default() - }) - .filter(tasks::Column::Id.is_in(pending_tasks.iter().map(|v| v.id.clone()))) - .exec(&conn) - .await?; - - for task in pending_tasks { - tasks_tx - .send(task) - .await - .map_err(|_error| anyhow::anyhow!("failed to send task to prefetch queue"))?; - } - } - } -} - -// Try to download metadata and remove task with asset_data update or update tasks to Pending/Failed -async fn execute_task(pool: PgPool, task: tasks::Model, timeout: Duration) -> anyhow::Result<()> { - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - match download_metadata(task.data, timeout).await { - Ok((asset_data_id, metadata)) => { - // Remove task and set metadata in transacstion - let txn = conn.begin().await?; - tasks::Entity::delete_by_id(task.id).exec(&txn).await?; - asset_data::Entity::update(asset_data::ActiveModel { - id: ActiveValue::Unchanged(asset_data_id), - metadata: ActiveValue::Set(metadata), - reindex: ActiveValue::Set(Some(false)), - ..Default::default() - }) - .exec(&txn) - .await?; - txn.commit().await?; - } - Err(error) => { - let status = if task.attempts + 1 == task.max_attempts { - TaskStatus::Failed - } else { - TaskStatus::Pending - }; - tasks::Entity::update(tasks::ActiveModel { - id: ActiveValue::Unchanged(task.id), - status: ActiveValue::Set(status), - attempts: ActiveValue::Set(task.attempts + 1), - errors: ActiveValue::Set(Some(error.to_string())), - ..Default::default() - }) - .exec(&conn) - .await?; - } - } - Ok(()) -} - -async fn download_metadata( - data: serde_json::Value, - timeout: Duration, -) -> anyhow::Result<(Vec, serde_json::Value)> { - let (id, uri, _slot) = serde_json::from_value::(data)?.into_inner(); - - // Need to check for malicious sites ? - let client = ClientBuilder::new().timeout(timeout).build()?; - let response = client.get(uri).send().await?; - - anyhow::ensure!( - response.status() == StatusCode::OK, - "HttpError status_code: {}", - response.status().as_str() - ); - Ok((id, response.json().await?)) -} diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 5212a589c..82f9237d5 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -1,26 +1,19 @@ use { crate::{ - config::{ConfigGrpc, REDIS_STREAM_DATA_KEY}, - prom::redis_xadd_status_inc, - redis::{metrics_xlen, TrackedPipeline}, + config::ConfigGrpc, prom::redis_xadd_status_inc, redis::TrackedPipeline, util::create_shutdown, }, anyhow::Context, - futures::{channel::mpsc, stream::StreamExt, SinkExt}, - redis::{streams::StreamMaxlen, RedisResult, Value as RedisValue}, + futures::stream::StreamExt, + redis::streams::StreamMaxlen, std::{collections::HashMap, sync::Arc, time::Duration}, - tokio::{ - spawn, - sync::Mutex, - task::JoinSet, - time::{sleep, Instant}, - }, + tokio::{sync::Mutex, time::sleep}, topograph::{ executor::{Executor, Nonblock, Tokio}, prelude::*, AsyncHandler, }, - tracing::{debug, info, warn}, + tracing::{debug, warn}, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::{SubscribeRequest, SubscribeUpdate}, @@ -118,7 +111,7 @@ impl<'a> AsyncHandler anyhow::Result<()> { +pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { let redis_client = redis::Client::open(config.redis.url.clone())?; let config = Arc::new(config); let connection = redis_client.get_multiplexed_tokio_connection().await?; @@ -142,17 +135,11 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { ..Default::default() }; - info!( - "cmd=grpc2redis request_snapshot={:?}", - config.request_snapshot - ); - let mut dragon_mouth_client = GeyserGrpcClient::build_from_shared(config.geyser_endpoint.clone())? .x_token(config.x_token.clone())? .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10)) - .set_x_request_snapshot(config.request_snapshot)? .connect() .await .context("failed to connect to gRPC")?; @@ -192,183 +179,3 @@ pub async fn run_v2(config: ConfigGrpc) -> anyhow::Result<()> { Ok(()) } - -#[allow(dead_code)] -pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { - let config = Arc::new(config); - let (tx, mut rx) = mpsc::channel::(config.geyser_update_message_buffer_size); // Adjust buffer size as needed - - // Connect to Redis - let client = redis::Client::open(config.redis.url.clone())?; - let connection = client.get_multiplexed_tokio_connection().await?; - - // Check stream length for the metrics - let jh_metrics_xlen = spawn({ - let connection = connection.clone(); - let streams = vec![ - config.accounts.stream.clone(), - config.transactions.stream.clone(), - ]; - async move { metrics_xlen(connection, &streams).await } - }); - tokio::pin!(jh_metrics_xlen); - - // Spawn gRPC client connections - let config = Arc::clone(&config); - let mut tx = tx.clone(); - - let mut client = GeyserGrpcClient::build_from_shared(config.geyser_endpoint.clone())? - .x_token(config.x_token.clone())? - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(10)) - .connect() - .await - .context("failed to connect to gRPC")?; - - let grc_config = Arc::clone(&config); - spawn(async move { - let mut accounts = HashMap::with_capacity(1); - let mut transactions = HashMap::with_capacity(1); - - accounts.insert( - "das".to_string(), - grc_config.accounts.filter.clone().to_proto(), - ); - transactions.insert( - "das".to_string(), - grc_config.transactions.filter.clone().to_proto(), - ); - - let request = SubscribeRequest { - accounts, - transactions, - ..Default::default() - }; - - let (_subscribe_tx, mut stream) = client.subscribe_with_request(Some(request)).await?; - - while let Some(Ok(msg)) = stream.next().await { - if let Some(update) = msg.update_oneof { - tx.send(update) - .await - .expect("Failed to send update to management thread"); - } - } - Ok::<(), anyhow::Error>(()) - }); - - // Management thread - let mut shutdown = create_shutdown()?; - let mut tasks = JoinSet::new(); - let mut pipe = redis::pipe(); - let mut pipe_accounts = 0; - let mut pipe_transactions = 0; - - let pipeline_max_idle = config.redis.pipeline_max_idle; - let deadline = sleep(pipeline_max_idle); - tokio::pin!(deadline); - - let result = loop { - tokio::select! { - result = &mut jh_metrics_xlen => match result { - Ok(Ok(_)) => unreachable!(), - Ok(Err(error)) => break Err(error), - Err(error) => break Err(error.into()), - }, - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - break Ok(()); - }, - Some(update) = rx.next() => { - match update { - UpdateOneof::Account(account) => { - - pipe.xadd_maxlen( - &config.accounts.stream, - StreamMaxlen::Approx(config.accounts.stream_maxlen), - "*", - &[(REDIS_STREAM_DATA_KEY, account.encode_to_vec())], - ); - - pipe_accounts += 1; - } - UpdateOneof::Transaction(transaction) => { - pipe.xadd_maxlen( - &config.transactions.stream, - StreamMaxlen::Approx(config.transactions.stream_maxlen), - "*", - &[(REDIS_STREAM_DATA_KEY, transaction.encode_to_vec())] - ); - - pipe_transactions += 1; - } - _ => continue, - } - if pipe_accounts + pipe_transactions >= config.redis.pipeline_max_size { - let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); - let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); - let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); - deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); - - tasks.spawn({ - let mut connection = connection.clone(); - let config = Arc::clone(&config); - async move { - let result: RedisResult = - pipe.atomic().query_async(&mut connection).await; - - let status = result.map(|_| ()).map_err(|_| ()); - redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); - redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); - - Ok::<(), anyhow::Error>(()) - } - }); - } - }, - _ = &mut deadline => { - if pipe_accounts + pipe_transactions > 0 { - let mut pipe = std::mem::replace(&mut pipe, redis::pipe()); - let pipe_accounts = std::mem::replace(&mut pipe_accounts, 0); - let pipe_transactions = std::mem::replace(&mut pipe_transactions, 0); - deadline.as_mut().reset(Instant::now() + config.redis.pipeline_max_idle); - - tasks.spawn({ - let mut connection = connection.clone(); - let config = Arc::clone(&config); - async move { - let result: RedisResult = - pipe.atomic().query_async(&mut connection).await; - - let status = result.map(|_| ()).map_err(|_| ()); - redis_xadd_status_inc(&config.accounts.stream, status, pipe_accounts); - redis_xadd_status_inc(&config.transactions.stream, status, pipe_transactions); - - Ok::<(), anyhow::Error>(()) - } - }); - } - }, - }; - - while tasks.len() >= config.redis.max_xadd_in_process { - if let Some(result) = tasks.join_next().await { - result??; - } - } - }; - - tokio::select! { - Some(signal) = shutdown.next() => { - anyhow::bail!("{signal} received, force shutdown..."); - } - result = async move { - while let Some(result) = tasks.join_next().await { - result??; - } - Ok::<(), anyhow::Error>(()) - } => result?, - }; - - result -} diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 30a7924f6..000f747b4 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,46 +1,17 @@ use { crate::{ - config::{ConfigIngester, ConfigIngesterDownloadMetadata, REDIS_STREAM_DATA_KEY}, - download_metadata::TASK_TYPE, - postgres::{create_pool as pg_create_pool, metrics_pgpool, report_pgpool}, - prom::{ - download_metadata_inserted_total_inc, program_transformer_task_status_inc, - program_transformer_tasks_total_set, redis_xack_inc, ProgramTransformerTaskStatusKind, - }, - redis::{ - metrics_xlen, IngestStream, ProgramTransformerInfo, RedisStream, RedisStreamMessage, - }, + config::{ConfigIngester, REDIS_STREAM_DATA_KEY}, + postgres::{create_pool as pg_create_pool, report_pgpool}, + prom::redis_xack_inc, + redis::{IngestStream, RedisStreamMessage}, util::create_shutdown, }, - chrono::Utc, - crypto::{digest::Digest, sha2::Sha256}, das_core::{DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier}, - digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, - futures::{ - future::{pending, BoxFuture, FusedFuture, FutureExt}, - stream::StreamExt, - }, - program_transformers::{ - error::ProgramTransformerError, AccountInfo, ProgramTransformer, TransactionInfo, - }, + futures::{future::BoxFuture, stream::StreamExt}, + program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, redis::aio::MultiplexedConnection, - sea_orm::{ - entity::{ActiveModelTrait, ActiveValue}, - error::{DbErr, RuntimeErr}, - SqlxPostgresConnector, - }, - sqlx::{Error as SqlxError, PgPool}, - std::{ - borrow::Cow, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - }, - tokio::{ - task::JoinSet, - time::{sleep, Duration}, - }, + std::sync::Arc, + tokio::time::{sleep, Duration}, tracing::warn, }; @@ -81,8 +52,8 @@ fn download_metadata_notifier_v2( ) } -pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { - let redis_client = redis::Client::open(config.redis.url.clone())?; +pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { + let redis_client = redis::Client::open(config.redis)?; let connection = redis_client.get_multiplexed_tokio_connection().await?; let pool = pg_create_pool(config.postgres).await?; @@ -91,12 +62,17 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { let accounts_download_metadata_notifier = download_metadata_notifier_v2( connection.clone(), - download_metadata_stream.clone(), + download_metadata_stream.name.clone(), + download_metadata_stream_maxlen, + )?; + let snapshots_download_metadata_notifier = download_metadata_notifier_v2( + connection.clone(), + download_metadata_stream.name.clone(), download_metadata_stream_maxlen, )?; let transactions_download_metadata_notifier = download_metadata_notifier_v2( connection.clone(), - download_metadata_stream.clone(), + download_metadata_stream.name.clone(), download_metadata_stream_maxlen, )?; @@ -104,6 +80,10 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { pool.clone(), accounts_download_metadata_notifier, )); + let pt_snapshots = Arc::new(ProgramTransformer::new( + pool.clone(), + snapshots_download_metadata_notifier, + )); let pt_transactions = Arc::new(ProgramTransformer::new( pool.clone(), transactions_download_metadata_notifier, @@ -114,7 +94,7 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); let download_metadata_stream = IngestStream::build() - .config(config.download_metadata.stream_config.clone()) + .config(config.download_metadata.stream.clone()) .connection(connection.clone()) .handler(move |info| { let download_metadata = Arc::clone(&download_metadata); @@ -161,258 +141,46 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { }) }) .start()?; + let snapshot_stream = IngestStream::build() + .config(config.snapshots.clone()) + .connection(connection.clone()) + .handler(move |info| { + let pt_snapshots = Arc::clone(&pt_snapshots); + + Box::pin(async move { + let info = AccountInfo::try_parse_msg(info)?; + + pt_snapshots + .handle_account_update(&info) + .await + .map_err(Into::into) + }) + }) + .start()?; let mut shutdown = create_shutdown()?; - loop { - tokio::select! { - _ = sleep(Duration::from_millis(100)) => { - report_pgpool(pool.clone()); - } - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - break; - } + let report_pool = pool.clone(); + let report_handle = tokio::spawn(async move { + let pool = report_pool.clone(); + loop { + sleep(Duration::from_millis(100)).await; + report_pgpool(pool.clone()); } + }); + + if let Some(signal) = shutdown.next().await { + warn!("{signal} received, waiting for spawned tasks..."); } + report_handle.abort(); + account_stream.stop().await?; transactions_stream.stop().await?; download_metadata_stream.stop().await?; + snapshot_stream.stop().await?; pool.close().await; Ok::<(), anyhow::Error>(()) } - -#[allow(dead_code)] -pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { - // connect to Redis - let client = redis::Client::open(config.redis.url.clone())?; - let connection = client.get_multiplexed_tokio_connection().await?; - - // check stream length for the metrics in spawned task - let jh_metrics_xlen = tokio::spawn({ - let connection = connection.clone(); - let streams = config - .redis - .streams - .iter() - .map(|config| config.stream.to_string()) - .collect::>(); - async move { metrics_xlen(connection, &streams).await } - }); - tokio::pin!(jh_metrics_xlen); - - // open connection to postgres - let pgpool = pg_create_pool(config.postgres).await?; - tokio::spawn({ - let pgpool = pgpool.clone(); - async move { metrics_pgpool(pgpool).await } - }); - - // create redis stream reader - let (mut redis_messages, redis_tasks_fut) = RedisStream::new(config.redis, connection).await?; - tokio::pin!(redis_tasks_fut); - - // program transforms related - let pt_accounts = Arc::new(ProgramTransformer::new( - pgpool.clone(), - create_download_metadata_notifier(pgpool.clone(), config.download_metadata.clone())?, - )); - let pt_transactions = Arc::new(ProgramTransformer::new( - pgpool.clone(), - create_download_metadata_notifier(pgpool.clone(), config.download_metadata)?, - )); - let pt_max_tasks_in_process = config.program_transformer.max_tasks_in_process; - let mut pt_tasks = JoinSet::new(); - let pt_tasks_len = Arc::new(AtomicUsize::new(0)); - - tokio::spawn({ - let pt_tasks_len = Arc::clone(&pt_tasks_len); - - async move { - loop { - program_transformer_tasks_total_set(pt_tasks_len.load(Ordering::Relaxed)); - sleep(Duration::from_millis(100)).await; - } - } - }); - - // read and process messages in the loop - let mut shutdown = create_shutdown()?; - loop { - pt_tasks_len.store(pt_tasks.len(), Ordering::Relaxed); - - let redis_messages_recv = if pt_tasks.len() == pt_max_tasks_in_process { - pending().boxed() - } else { - redis_messages.recv().boxed() - }; - let pt_tasks_next = if pt_tasks.is_empty() { - pending().boxed() - } else { - pt_tasks.join_next().boxed() - }; - - let msg = tokio::select! { - result = &mut jh_metrics_xlen => match result { - Ok(Ok(_)) => unreachable!(), - Ok(Err(error)) => break Err(error), - Err(error) => break Err(error.into()), - }, - Some(signal) = shutdown.next() => { - warn!("{signal} received, waiting spawned tasks..."); - break Ok(()); - }, - result = &mut redis_tasks_fut => break result, - msg = redis_messages_recv => match msg { - Some(msg) => msg, - None => break Ok(()), - }, - result = pt_tasks_next => { - if let Some(result) = result { - result??; - } - continue; - } - }; - - pt_tasks.spawn({ - let pt_accounts = Arc::clone(&pt_accounts); - let pt_transactions = Arc::clone(&pt_transactions); - async move { - let result = match &msg.get_data() { - ProgramTransformerInfo::Account(account) => { - pt_accounts.handle_account_update(account).await - } - ProgramTransformerInfo::Transaction(transaction) => { - pt_transactions.handle_transaction(transaction).await - } - ProgramTransformerInfo::MetadataJson(_download_metadata_info) => Ok(()), - }; - - macro_rules! log_or_bail { - ($action:path, $msg:expr, $error:ident) => { - match msg.get_data() { - ProgramTransformerInfo::Account(account) => { - $action!("{} account {}: {:?}", $msg, account.pubkey, $error) - } - ProgramTransformerInfo::Transaction(transaction) => { - $action!( - "{} transaction {}: {:?}", - $msg, - transaction.signature, - $error - ) - } - ProgramTransformerInfo::MetadataJson(_download_metadata_info) => { - todo!() - } - } - }; - } - - match result { - Ok(()) => program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::Success, - ), - Err(ProgramTransformerError::NotImplemented) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::NotImplemented, - ) - } - Err(ProgramTransformerError::DeserializationError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::DeserializationError, - ); - log_or_bail!(warn, "failed to deserialize", error) - } - Err(ProgramTransformerError::ParsingError(error)) => { - program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::ParsingError, - ); - log_or_bail!(warn, "failed to parse", error) - } - Err(ProgramTransformerError::DatabaseError(error)) => { - log_or_bail!(anyhow::bail, "database error for", error) - } - Err(ProgramTransformerError::AssetIndexError(error)) => { - log_or_bail!(anyhow::bail, "indexing error for ", error) - } - Err(error) => { - log_or_bail!(anyhow::bail, "failed to handle", error) - } - } - - msg.ack() - } - }); - }?; - - tokio::select! { - Some(signal) = shutdown.next() => { - anyhow::bail!("{signal} received, force shutdown..."); - } - result = async move { - // shutdown `prefetch` channel (but not Receiver) - redis_messages.shutdown(); - // wait all `program_transformer` spawned tasks - while let Some(result) = pt_tasks.join_next().await { - result??; - } - // wait all `ack` spawned tasks - if !redis_tasks_fut.is_terminated() { - redis_tasks_fut.await?; - } - // shutdown database connection - pgpool.close().await; - Ok::<(), anyhow::Error>(()) - } => result, - } -} - -fn create_download_metadata_notifier( - pgpool: PgPool, - config: ConfigIngesterDownloadMetadata, -) -> anyhow::Result { - let max_attempts = config.max_attempts.try_into()?; - Ok(Box::new(move |info: DownloadMetadataInfo| -> BoxFuture< - 'static, - Result<(), Box>, - > { - let pgpool = pgpool.clone(); - Box::pin(async move { - let data = serde_json::to_value(info)?; - - let mut hasher = Sha256::new(); - hasher.input(TASK_TYPE.as_bytes()); - hasher.input(serde_json::to_vec(&data)?.as_slice()); - let hash = hasher.result_str(); - - let model = tasks::ActiveModel { - id: ActiveValue::Set(hash), - task_type: ActiveValue::Set(TASK_TYPE.to_owned()), - data: ActiveValue::Set(data), - status: ActiveValue::Set(TaskStatus::Pending), - created_at: ActiveValue::Set(Utc::now().naive_utc()), - locked_until: ActiveValue::Set(None), - locked_by: ActiveValue::Set(None), - max_attempts: ActiveValue::Set(max_attempts), - attempts: ActiveValue::Set(0), - duration: ActiveValue::Set(None), - errors: ActiveValue::Set(None), - }; - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pgpool); - - match model.insert(&conn).await.map(|_mode| ()) { - // skip unique_violation error - Err(DbErr::Query(RuntimeErr::SqlxError(SqlxError::Database(dberr)))) if dberr.code() == Some(Cow::Borrowed("23505")) => {}, - value => value?, - }; - download_metadata_inserted_total_inc(); - - Ok(()) - }) - })) -} diff --git a/grpc-ingest/src/main.rs b/grpc-ingest/src/main.rs index f467d3961..907e1e00c 100644 --- a/grpc-ingest/src/main.rs +++ b/grpc-ingest/src/main.rs @@ -1,9 +1,6 @@ use { crate::{ - config::{ - load as config_load, ConfigDownloadMetadata, ConfigGrpc, ConfigIngester, - ConfigPrometheus, - }, + config::{load as config_load, ConfigGrpc, ConfigIngester, ConfigPrometheus}, prom::run_server as prometheus_run_server, tracing::init as tracing_init, }, @@ -13,7 +10,6 @@ use { }; mod config; -mod download_metadata; mod grpc; mod ingester; mod postgres; @@ -46,9 +42,6 @@ enum ArgsAction { /// Run ingester process (process events from Redis) #[command(name = "ingester")] Ingester, - /// Run metadata downloader - #[command(name = "download-metadata")] - DownloadMetadata, } #[tokio::main] @@ -71,19 +64,13 @@ async fn main() -> anyhow::Result<()> { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; - grpc::run_v2(config).await + grpc::run(config).await } ArgsAction::Ingester => { let config = config_load::(&args.config) .await .with_context(|| format!("failed to parse config from: {}", args.config))?; - ingester::run_v2(config).await - } - ArgsAction::DownloadMetadata => { - let config = config_load::(&args.config) - .await - .with_context(|| format!("failed to parse config from: {}", args.config))?; - download_metadata::run(config).await + ingester::run(config).await } } } diff --git a/grpc-ingest/src/postgres.rs b/grpc-ingest/src/postgres.rs index dd29ca730..07e0ff929 100644 --- a/grpc-ingest/src/postgres.rs +++ b/grpc-ingest/src/postgres.rs @@ -7,7 +7,6 @@ use { postgres::{PgConnectOptions, PgPoolOptions}, PgPool, }, - tokio::time::{sleep, Duration}, }; pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result { @@ -24,10 +23,3 @@ pub fn report_pgpool(pgpool: PgPool) { pgpool_connections_set(PgpoolConnectionsKind::Total, pgpool.size() as usize); pgpool_connections_set(PgpoolConnectionsKind::Idle, pgpool.num_idle()); } - -pub async fn metrics_pgpool(pgpool: PgPool) { - loop { - report_pgpool(pgpool.clone()); - sleep(Duration::from_millis(100)).await; - } -} diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 83b51011c..33f2c6220 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -162,10 +162,6 @@ pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { .set(size as i64) } -pub fn program_transformer_tasks_total_set(size: usize) { - PROGRAM_TRANSFORMER_TASKS_TOTAL.set(size as i64) -} - pub fn ingest_tasks_total_inc(stream: &str) { INGEST_TASKS_TOTAL.with_label_values(&[stream]).inc() } @@ -287,7 +283,3 @@ pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKin .with_label_values(&[kind.to_str()]) .inc() } - -pub fn download_metadata_inserted_total_inc() { - DOWNLOAD_METADATA_INSERTED_TOTAL.inc() -} diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index feef7228c..8510683c4 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -1,9 +1,6 @@ use { crate::{ - config::{ - ConfigIngestStream, ConfigIngesterRedis, ConfigIngesterRedisStreamType, - REDIS_STREAM_DATA_KEY, - }, + config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, prom::{ ingest_tasks_reset, ingest_tasks_total_dec, ingest_tasks_total_inc, program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, @@ -11,7 +8,7 @@ use { }, }, das_core::DownloadMetadataInfo, - futures::future::{BoxFuture, Fuse, FutureExt}, + futures::future::BoxFuture, program_transformers::{AccountInfo, TransactionInfo}, redis::{ aio::MultiplexedConnection, @@ -22,19 +19,8 @@ use { AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, solana_sdk::{pubkey::Pubkey, signature::Signature}, - std::{ - collections::HashMap, - convert::Infallible, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - }, - tokio::{ - sync::mpsc, - task::JoinSet, - time::{sleep, Duration, Instant}, - }, + std::{collections::HashMap, sync::Arc}, + tokio::time::{sleep, Duration}, topograph::{ executor::{Executor, Nonblock, Tokio}, prelude::*, @@ -660,24 +646,6 @@ pub async fn report_xlen( Ok(()) } -pub async fn metrics_xlen( - mut connection: C, - streams: &[String], -) -> anyhow::Result { - loop { - let mut pipe = redis::pipe(); - for stream in streams { - pipe.xlen(stream); - } - let xlens: Vec = pipe.query_async(&mut connection).await?; - - for (stream, xlen) in streams.iter().zip(xlens.into_iter()) { - redis_xlen_set(stream, xlen); - } - - sleep(Duration::from_millis(100)).await; - } -} pub async fn xgroup_create( connection: &mut C, @@ -707,463 +675,6 @@ pub async fn xgroup_create( Ok(()) } -#[derive(Debug)] -struct RedisStreamInfo { - group: String, - consumer: String, - stream_name: String, - stream_type: ConfigIngesterRedisStreamType, - xack_batch_max_size: usize, - xack_batch_max_idle: Duration, - xack_max_in_process: usize, -} - -#[derive(Debug)] -pub enum ProgramTransformerInfo { - Account(AccountInfo), - Transaction(TransactionInfo), - MetadataJson(DownloadMetadataInfo), -} - -#[derive(Debug)] -pub struct RedisStreamMessageInfo { - id: String, - data: ProgramTransformerInfo, - ack_tx: mpsc::UnboundedSender, -} - -impl RedisStreamMessageInfo { - fn parse( - stream: &RedisStreamInfo, - StreamId { id, map }: StreamId, - ack_tx: mpsc::UnboundedSender, - ) -> anyhow::Result { - let to_anyhow = |error: String| anyhow::anyhow!(error); - - let data = match map.get(REDIS_STREAM_DATA_KEY) { - Some(RedisValue::Data(vec)) => match stream.stream_type { - ConfigIngesterRedisStreamType::Account => { - let SubscribeUpdateAccount { account, slot, .. } = - Message::decode(vec.as_ref())?; - - let account = account.ok_or_else(|| { - anyhow::anyhow!("received invalid SubscribeUpdateAccount") - })?; - - ProgramTransformerInfo::Account(AccountInfo { - slot, - pubkey: Pubkey::try_from(account.pubkey.as_slice())?, - owner: Pubkey::try_from(account.owner.as_slice())?, - data: account.data, - }) - } - ConfigIngesterRedisStreamType::Transaction => { - let SubscribeUpdateTransaction { transaction, slot } = - Message::decode(vec.as_ref())?; - - let transaction = transaction.ok_or_else(|| { - anyhow::anyhow!("received invalid SubscribeUpdateTransaction") - })?; - let tx = transaction.transaction.ok_or_else(|| { - anyhow::anyhow!( - "received invalid transaction in SubscribeUpdateTransaction" - ) - })?; - let message = tx.message.ok_or_else(|| { - anyhow::anyhow!("received invalid message in SubscribeUpdateTransaction") - })?; - let meta = transaction.meta.ok_or_else(|| { - anyhow::anyhow!("received invalid meta in SubscribeUpdateTransaction") - })?; - - let mut account_keys = - create_pubkey_vec(message.account_keys).map_err(to_anyhow)?; - for pubkey in - create_pubkey_vec(meta.loaded_writable_addresses).map_err(to_anyhow)? - { - account_keys.push(pubkey); - } - for pubkey in - create_pubkey_vec(meta.loaded_readonly_addresses).map_err(to_anyhow)? - { - account_keys.push(pubkey); - } - - ProgramTransformerInfo::Transaction(TransactionInfo { - slot, - signature: Signature::try_from(transaction.signature.as_slice())?, - account_keys, - message_instructions: create_message_instructions(message.instructions) - .map_err(to_anyhow)?, - meta_inner_instructions: create_meta_inner_instructions( - meta.inner_instructions, - ) - .map_err(to_anyhow)?, - }) - } - ConfigIngesterRedisStreamType::MetadataJson => { - let info: DownloadMetadataInfo = serde_json::from_slice(vec.as_ref())?; - - ProgramTransformerInfo::MetadataJson(info) - } - }, - Some(_) => anyhow::bail!( - "invalid data (key: {:?}) from stream {:?}", - REDIS_STREAM_DATA_KEY, - stream.stream_name - ), - None => anyhow::bail!( - "failed to get data (key: {:?}) from stream {:?}", - REDIS_STREAM_DATA_KEY, - stream.stream_name - ), - }; - Ok(Self { id, data, ack_tx }) - } - - pub const fn get_data(&self) -> &ProgramTransformerInfo { - &self.data - } - - pub fn ack(self) -> anyhow::Result<()> { - self.ack_tx - .send(self.id) - .map_err(|_error| anyhow::anyhow!("failed to send message to ack channel")) - } -} - -#[derive(Debug)] -pub struct RedisStream { - shutdown: Arc, - messages_rx: mpsc::Receiver, -} - -#[allow(dead_code)] -async fn run_ack( - stream: Arc, - connection: MultiplexedConnection, - mut ack_rx: mpsc::UnboundedReceiver, -) -> anyhow::Result<()> { - let mut ids = vec![]; - let deadline = sleep(stream.xack_batch_max_idle); - tokio::pin!(deadline); - let mut tasks = JoinSet::new(); - - let result = loop { - let terminated = tokio::select! { - msg = ack_rx.recv() => match msg { - Some(msg) => { - ids.push(msg); - if ids.len() < stream.xack_batch_max_size { - continue; - } - false - } - None => true, - }, - _ = &mut deadline => false, - }; - - let ids = std::mem::take(&mut ids); - deadline - .as_mut() - .reset(Instant::now() + stream.xack_batch_max_idle); - if !ids.is_empty() { - tasks.spawn({ - let stream = Arc::clone(&stream); - let mut connection = connection.clone(); - async move { - match redis::pipe() - .atomic() - .xack(&stream.stream_name, &stream.group, &ids) - .xdel(&stream.stream_name, &ids) - .query_async::<_, redis::Value>(&mut connection) - .await - { - Ok(_) => { - debug!("Acknowledged and deleted idle messages: {:?}", ids); - redis_xack_inc(&stream.stream_name, ids.len()); - } - Err(e) => { - error!("Failed to acknowledge or delete idle messages: {:?}", e); - } - } - redis_xack_inc(&stream.stream_name, ids.len()); - Ok::<(), anyhow::Error>(()) - } - }); - while tasks.len() >= stream.xack_max_in_process { - if let Some(result) = tasks.join_next().await { - result??; - } - } - } - - if terminated { - break Ok(()); - } - }; - - while let Some(result) = tasks.join_next().await { - result??; - } - - result -} - -impl RedisStream { - pub async fn new( - config: ConfigIngesterRedis, - mut connection: MultiplexedConnection, - ) -> anyhow::Result<(Self, Fuse>>)> { - // create group with consumer per stream - for stream in config.streams.iter() { - xgroup_create( - &mut connection, - stream.stream, - &config.group, - &config.consumer, - ) - .await?; - } - - // shutdown flag - let shutdown = Arc::new(AtomicBool::new(false)); - - // create stream info wrapped by Arc - let mut ack_tasks = vec![]; - let streams = config - .streams - .iter() - .map(|stream| { - let (ack_tx, ack_rx) = mpsc::unbounded_channel(); - let info = Arc::new(RedisStreamInfo { - group: config.group.clone(), - consumer: config.consumer.clone(), - stream_name: stream.stream.to_string(), - stream_type: stream.stream_type, - xack_batch_max_size: stream.xack_batch_max_size, - xack_batch_max_idle: stream.xack_batch_max_idle, - xack_max_in_process: stream.xack_max_in_process, - }); - ack_tasks.push((Arc::clone(&info), ack_rx)); - (stream.stream, (ack_tx, info)) - }) - .collect::>(); - - // spawn xack tasks - let ack_jh_vec = ack_tasks - .into_iter() - .map(|(stream, ack_rx)| { - let connection = connection.clone(); - tokio::spawn(async move { Self::run_ack(stream, connection, ack_rx).await }) - }) - .collect::>(); - - // spawn prefetch task - let (messages_tx, messages_rx) = mpsc::channel(config.prefetch_queue_size); - let jh_prefetch = tokio::spawn({ - let shutdown = Arc::clone(&shutdown); - async move { Self::run_prefetch(config, streams, connection, messages_tx, shutdown).await } - }); - - // merge spawned xack / prefetch tasks - let spawned_tasks = async move { - jh_prefetch.await??; - for jh in ack_jh_vec.into_iter() { - jh.await??; - } - Ok::<(), anyhow::Error>(()) - }; - - Ok(( - Self { - shutdown, - messages_rx, - }, - spawned_tasks.boxed().fuse(), - )) - } - - pub async fn recv(&mut self) -> Option { - self.messages_rx.recv().await - } - - pub fn shutdown(mut self) { - self.shutdown.store(true, Ordering::Relaxed); - tokio::spawn(async move { while self.messages_rx.recv().await.is_some() {} }); - } - - async fn run_prefetch( - config: ConfigIngesterRedis, - streams: HashMap<&str, (mpsc::UnboundedSender, Arc)>, - mut connection: MultiplexedConnection, - messages_tx: mpsc::Sender, - shutdown: Arc, - ) -> anyhow::Result<()> { - // read pending first - for (ack_tx, stream) in streams.values() { - let mut start = "-".to_owned(); - while !shutdown.load(Ordering::Relaxed) { - let StreamPendingCountReply { ids: pending_ids } = redis::cmd("XPENDING") - .arg(&stream.stream_name) - .arg(&stream.group) - .arg(&start) - .arg("+") - .arg(config.xpending_max) - .arg(&stream.consumer) // we can't use `xpending_count` because it doesn't support `consumer` filter - .query_async(&mut connection) - .await?; - - // drop first item if we do not start from the beginning - let used_ids = if start == "-" { 0.. } else { 1.. }; - let ids_str = pending_ids[used_ids] - .iter() - .map(|pending| pending.id.as_str()) - .collect::>(); - - // check that we fetched all pendings and update start - match pending_ids.last() { - Some(id) => { - if id.id == start { - break; - } else { - start = id.id.clone(); - } - } - None => break, - } - - let StreamClaimReply { ids: pendings } = connection - .xclaim( - &stream.stream_name, - &stream.group, - &stream.consumer, - 0, - &ids_str, - ) - .await?; - for pending in pendings { - let item = RedisStreamMessageInfo::parse(stream, pending, ack_tx.clone())?; - messages_tx.send(item).await.map_err(|_error| { - anyhow::anyhow!("failed to send item to prefetch channel") - })?; - } - } - } - - // exit if need to handle only pending - if config.xpending_only { - return Ok(()); - } - - let streams_keys = streams.keys().collect::>(); - let streams_ids = (0..streams_keys.len()).map(|_| ">").collect::>(); - - while !shutdown.load(Ordering::Relaxed) { - let opts = StreamReadOptions::default() - .count(config.xreadgroup_max) - .group(&config.group, &config.consumer); - let results: StreamReadReply = connection - .xread_options(&streams_keys, &streams_ids, &opts) - .await?; - - if results.keys.is_empty() { - sleep(Duration::from_millis(5)).await; - continue; - } - - for StreamKey { key, ids } in results.keys { - let (ack_tx, stream) = match streams.get(key.as_str()) { - Some(value) => value, - None => anyhow::bail!("unknown stream: {:?}", key), - }; - - for id in ids { - let item = RedisStreamMessageInfo::parse(stream, id, ack_tx.clone())?; - messages_tx.send(item).await.map_err(|_error| { - anyhow::anyhow!("failed to send item to prefetch channel") - })?; - } - } - } - - Ok(()) - } - - async fn run_ack( - stream: Arc, - connection: MultiplexedConnection, - mut ack_rx: mpsc::UnboundedReceiver, - ) -> anyhow::Result<()> { - let mut ids = vec![]; - let deadline = sleep(stream.xack_batch_max_idle); - tokio::pin!(deadline); - let mut tasks = JoinSet::new(); - - let result = loop { - let terminated = tokio::select! { - msg = ack_rx.recv() => match msg { - Some(msg) => { - ids.push(msg); - if ids.len() < stream.xack_batch_max_size { - continue; - } - false - } - None => true, - }, - _ = &mut deadline => false, - }; - - let ids = std::mem::take(&mut ids); - deadline - .as_mut() - .reset(Instant::now() + stream.xack_batch_max_idle); - if !ids.is_empty() { - tasks.spawn({ - let stream = Arc::clone(&stream); - let mut connection = connection.clone(); - async move { - match redis::pipe() - .atomic() - .xack(&stream.stream_name, &stream.group, &ids) - .xdel(&stream.stream_name, &ids) - .query_async::<_, redis::Value>(&mut connection) - .await - { - Ok(info) => { - info!("Acknowledged and deleted idle messages: {:?}", info); - redis_xack_inc(&stream.stream_name, ids.len()); - } - Err(e) => { - error!("Failed to acknowledge or delete idle messages: {:?}", e); - } - } - redis_xack_inc(&stream.stream_name, ids.len()); - Ok::<(), anyhow::Error>(()) - } - }); - while tasks.len() >= stream.xack_max_in_process { - if let Some(result) = tasks.join_next().await { - result??; - } - } - } - - if terminated { - break Ok(()); - } - }; - - while let Some(result) = tasks.join_next().await { - result??; - } - - result - } -} - pub struct TrackedPipeline { pipeline: redis::Pipeline, counts: HashMap, From 118baec1d42d4068d86f0a4f572e8103b0bda8c6 Mon Sep 17 00:00:00 2001 From: Nagaprasad V R <81755170+Nagaprasadvr@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:28:10 +0530 Subject: [PATCH 48/72] fix: keep grpc alive by sending using ping-pong (#160) --- grpc-ingest/src/grpc.rs | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 82f9237d5..59eda0492 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -4,9 +4,9 @@ use { util::create_shutdown, }, anyhow::Context, - futures::stream::StreamExt, + futures::{channel::mpsc::SendError, stream::StreamExt, Sink, SinkExt}, redis::streams::StreamMaxlen, - std::{collections::HashMap, sync::Arc, time::Duration}, + std::{collections::HashMap, pin::Pin, sync::Arc, time::Duration}, tokio::{sync::Mutex, time::sleep}, topograph::{ executor::{Executor, Nonblock, Tokio}, @@ -16,23 +16,28 @@ use { tracing::{debug, warn}, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ - geyser::{SubscribeRequest, SubscribeUpdate}, + geyser::{SubscribeRequest, SubscribeRequestPing, SubscribeUpdate}, prelude::subscribe_update::UpdateOneof, prost::Message, }, yellowstone_grpc_tools::config::GrpcRequestToProto, }; +const PING_ID: i32 = 0; + enum GrpcJob { FlushRedisPipe, ProcessSubscribeUpdate(Box), } +type SubscribeTx = Pin + Send + Sync>>; + #[derive(Clone)] pub struct GrpcJobHandler { connection: redis::aio::MultiplexedConnection, config: Arc, pipe: Arc>, + subscribe_tx: Arc>, } impl<'a> AsyncHandler>> @@ -49,6 +54,8 @@ impl<'a> AsyncHandler { @@ -98,6 +105,29 @@ impl<'a> AsyncHandler { + subscribe_tx + .lock() + .await + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: PING_ID }), + ..Default::default() + }) + .await + .map_err(|err| { + warn!(message = "Failed to send ping", ?err); + }) + .ok(); + + debug!(message = "Ping", id = PING_ID); + } + UpdateOneof::Pong(pong) => { + if pong.id == PING_ID { + debug!(message = "Pong", id = PING_ID); + } else { + warn!(message = "Unknown pong id", id = pong.id); + } + } var => warn!(message = "Unknown update variant", ?var), } } @@ -144,9 +174,10 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { .await .context("failed to connect to gRPC")?; - let (_subscribe_tx, stream) = dragon_mouth_client + let (subscribe_tx, stream) = dragon_mouth_client .subscribe_with_request(Some(request)) .await?; + tokio::pin!(stream); let exec = Executor::builder(Nonblock(Tokio)) @@ -155,6 +186,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { config: Arc::clone(&config), connection: connection.clone(), pipe: Arc::clone(&pipe), + subscribe_tx: Arc::new(Mutex::new(Box::pin(subscribe_tx))), })?; let deadline_config = Arc::clone(&config); From 01bbe9961920463ecb3bf7137bf0044cb4dc45d5 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 9 Oct 2024 17:34:46 +0200 Subject: [PATCH 49/72] Only Time Based Flushes (#161) * Include prometheus metric explorer * Only xadd flush on a timeer * Follow prom metric naming conventions * Logging changes for ping-pong grpc --- docker-compose.yaml | 10 +++++ grpc-ingest/config-grpc2redis.yml | 5 +-- grpc-ingest/src/config.rs | 21 +-------- grpc-ingest/src/grpc.rs | 37 +++++++--------- grpc-ingest/src/prom.rs | 72 +++++++++++++++---------------- grpc-ingest/src/redis.rs | 4 -- prometheus-config.yaml | 9 ++++ 7 files changed, 73 insertions(+), 85 deletions(-) create mode 100644 prometheus-config.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index 364a00855..8c44dc124 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -112,6 +112,16 @@ services: - "8001:8001" - "8899:8899" - "9900:9900" + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus-config.yaml:/etc/prometheus/prometheus-config.yaml + command: ["--config.file=/etc/prometheus/prometheus-config.yaml"] + ports: + - "9091:9090" + extra_hosts: + - "host.docker.internal:host-gateway" volumes: grafana_data: { } graphite_data: { } diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index 0a7fef2d0..8ec202637 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -22,6 +22,5 @@ transactions: - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY redis: url: redis://localhost:6379 - pipeline_max_size: 10 - pipeline_max_idle_ms: 10 - max_xadd_in_process: 100 + pipeline_max_size: 1_000 + pipeline_max_idle_ms: 100_000 diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 5d998bb4f..65c28c658 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -181,36 +181,17 @@ impl ConfigGrpcTransactions { #[derive(Debug, Clone, Deserialize)] pub struct ConfigGrpcRedis { pub url: String, - #[serde( - default = "ConfigGrpcRedis::default_pipeline_max_size", - deserialize_with = "deserialize_usize_str" - )] - pub pipeline_max_size: usize, #[serde( default = "ConfigGrpcRedis::default_pipeline_max_idle", - deserialize_with = "deserialize_duration_str", - rename = "pipeline_max_idle_ms" + deserialize_with = "deserialize_duration_str" )] pub pipeline_max_idle: Duration, - #[serde( - default = "ConfigGrpcRedis::default_max_xadd_in_process", - deserialize_with = "deserialize_usize_str" - )] - pub max_xadd_in_process: usize, } impl ConfigGrpcRedis { - pub const fn default_pipeline_max_size() -> usize { - 10 - } - pub const fn default_pipeline_max_idle() -> Duration { Duration::from_millis(10) } - - pub const fn default_max_xadd_in_process() -> usize { - 100 - } } pub fn deserialize_duration_str<'de, D>(deserializer: D) -> Result diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 59eda0492..6529f5194 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -48,7 +48,7 @@ impl<'a> AsyncHandler>, + _handle: topograph::executor::Handle<'a, GrpcJob, Nonblock>, ) -> impl futures::Future + Send + 'a { let config = Arc::clone(&self.config); let connection = self.connection.clone(); @@ -68,10 +68,9 @@ impl<'a> AsyncHandler { let accounts_stream = config.accounts.stream.clone(); @@ -92,8 +91,6 @@ impl<'a> AsyncHandler { pipe.xadd_maxlen( @@ -102,39 +99,36 @@ impl<'a> AsyncHandler { - subscribe_tx + let ping = subscribe_tx .lock() .await .send(SubscribeRequest { ping: Some(SubscribeRequestPing { id: PING_ID }), ..Default::default() }) - .await - .map_err(|err| { - warn!(message = "Failed to send ping", ?err); - }) - .ok(); - - debug!(message = "Ping", id = PING_ID); + .await; + + match ping { + Ok(_) => { + debug!(message = "Ping sent successfully", id = PING_ID) + } + Err(err) => { + warn!(message = "Failed to send ping", ?err, id = PING_ID) + } + } } UpdateOneof::Pong(pong) => { if pong.id == PING_ID { - debug!(message = "Pong", id = PING_ID); + debug!(message = "Pong received", id = PING_ID); } else { - warn!(message = "Unknown pong id", id = pong.id); + warn!(message = "Unknown pong id received", id = pong.id); } } var => warn!(message = "Unknown update variant", ?var), } } - - if pipe.size() >= config.redis.pipeline_max_size { - handle.push(GrpcJob::FlushRedisPipe); - } } } } @@ -197,7 +191,6 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { exec.push(GrpcJob::FlushRedisPipe); } Some(Ok(msg)) = stream.next() => { - debug!(message = "Received gRPC message", ?msg); exec.push(GrpcJob::ProcessSubscribeUpdate(Box::new(msg))); } _ = shutdown.next() => { diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 33f2c6220..aaa8ac273 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -15,46 +15,46 @@ use { lazy_static::lazy_static! { static ref REGISTRY: Registry = Registry::new(); - static ref VERSION: IntCounterVec = IntCounterVec::new( - Opts::new("version", "Plugin version info"), + static ref VERSION_INFO_METRIC: IntCounterVec = IntCounterVec::new( + Opts::new("version_info", "Plugin version info"), &["buildts", "git", "package", "proto", "rustc", "solana", "version"] ).unwrap(); - static ref REDIS_XLEN_TOTAL: IntGaugeVec = IntGaugeVec::new( - Opts::new("redis_xlen_total", "Length of stream in Redis"), + static ref REDIS_STREAM_LENGTH: IntGaugeVec = IntGaugeVec::new( + Opts::new("redis_stream_length", "Length of stream in Redis"), &["stream"] ).unwrap(); - static ref REDIS_XADD_STATUS: IntCounterVec = IntCounterVec::new( - Opts::new("redis_xadd_status", "Status of messages sent to Redis stream"), + static ref REDIS_XADD_STATUS_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xadd_status_count", "Status of messages sent to Redis stream"), &["stream", "status"] ).unwrap(); - static ref REDIS_XACK_TOTAL: IntCounterVec = IntCounterVec::new( - Opts::new("redis_xack_total", "Total number of processed messages"), + static ref REDIS_XACK_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xack_count", "Total number of processed messages"), &["stream"] ).unwrap(); - static ref PGPOOL_CONNECTIONS_TOTAL: IntGaugeVec = IntGaugeVec::new( - Opts::new("pgpool_connections_total", "Total number of connections in Postgres Pool"), + static ref PGPOOL_CONNECTIONS: IntGaugeVec = IntGaugeVec::new( + Opts::new("pgpool_connections", "Total number of connections in Postgres Pool"), &["kind"] ).unwrap(); - static ref PROGRAM_TRANSFORMER_TASKS_TOTAL: IntGauge = IntGauge::new( - "program_transformer_tasks_total", "Number of tasks spawned for program transform" + static ref PROGRAM_TRANSFORMER_TASKS: IntGauge = IntGauge::new( + "program_transformer_tasks", "Number of tasks spawned for program transform" ).unwrap(); - static ref PROGRAM_TRANSFORMER_TASK_STATUS: IntCounterVec = IntCounterVec::new( - Opts::new("program_transformer_task_status", "Status of processed messages"), + static ref PROGRAM_TRANSFORMER_TASK_STATUS_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("program_transformer_task_status_count", "Status of processed messages"), &["status"], ).unwrap(); - static ref DOWNLOAD_METADATA_INSERTED_TOTAL: IntCounter = IntCounter::new( - "download_metadata_inserted_total", "Total number of inserted tasks for download metadata" + static ref DOWNLOAD_METADATA_INSERTED_COUNT: IntCounter = IntCounter::new( + "download_metadata_inserted_count", "Total number of inserted tasks for download metadata" ).unwrap(); - static ref INGEST_TASKS_TOTAL: IntGaugeVec = IntGaugeVec::new( - Opts::new("ingest_tasks_total", "Number of tasks spawned for ingest"), + static ref INGEST_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("ingest_tasks", "Number of tasks spawned for ingest"), &["stream"] ).unwrap(); } @@ -69,17 +69,17 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { .expect("collector can't be registered"); }; } - register!(VERSION); - register!(REDIS_XLEN_TOTAL); - register!(REDIS_XADD_STATUS); - register!(REDIS_XACK_TOTAL); - register!(PGPOOL_CONNECTIONS_TOTAL); - register!(PROGRAM_TRANSFORMER_TASKS_TOTAL); - register!(PROGRAM_TRANSFORMER_TASK_STATUS); - register!(DOWNLOAD_METADATA_INSERTED_TOTAL); - register!(INGEST_TASKS_TOTAL); + register!(VERSION_INFO_METRIC); + register!(REDIS_STREAM_LENGTH); + register!(REDIS_XADD_STATUS_COUNT); + register!(REDIS_XACK_COUNT); + register!(PGPOOL_CONNECTIONS); + register!(PROGRAM_TRANSFORMER_TASKS); + register!(PROGRAM_TRANSFORMER_TASK_STATUS_COUNT); + register!(DOWNLOAD_METADATA_INSERTED_COUNT); + register!(INGEST_TASKS); - VERSION + VERSION_INFO_METRIC .with_label_values(&[ VERSION_INFO.buildts, VERSION_INFO.git, @@ -130,19 +130,19 @@ fn not_found_handler() -> Response { } pub fn redis_xlen_set(stream: &str, len: usize) { - REDIS_XLEN_TOTAL + REDIS_STREAM_LENGTH .with_label_values(&[stream]) .set(len as i64); } pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) { - REDIS_XADD_STATUS + REDIS_XADD_STATUS_COUNT .with_label_values(&[stream, if status.is_ok() { "success" } else { "failed" }]) .inc_by(delta as u64); } pub fn redis_xack_inc(stream: &str, delta: usize) { - REDIS_XACK_TOTAL + REDIS_XACK_COUNT .with_label_values(&[stream]) .inc_by(delta as u64) } @@ -154,7 +154,7 @@ pub enum PgpoolConnectionsKind { } pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { - PGPOOL_CONNECTIONS_TOTAL + PGPOOL_CONNECTIONS .with_label_values(&[match kind { PgpoolConnectionsKind::Total => "total", PgpoolConnectionsKind::Idle => "idle", @@ -163,15 +163,15 @@ pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { } pub fn ingest_tasks_total_inc(stream: &str) { - INGEST_TASKS_TOTAL.with_label_values(&[stream]).inc() + INGEST_TASKS.with_label_values(&[stream]).inc() } pub fn ingest_tasks_total_dec(stream: &str) { - INGEST_TASKS_TOTAL.with_label_values(&[stream]).dec() + INGEST_TASKS.with_label_values(&[stream]).dec() } pub fn ingest_tasks_reset(stream: &str) { - INGEST_TASKS_TOTAL.with_label_values(&[stream]).set(0) + INGEST_TASKS.with_label_values(&[stream]).set(0) } #[derive(Debug, Clone, Copy)] @@ -279,7 +279,7 @@ impl ProgramTransformerTaskStatusKind { } pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKind) { - PROGRAM_TRANSFORMER_TASK_STATUS + PROGRAM_TRANSFORMER_TASK_STATUS_COUNT .with_label_values(&[kind.to_str()]) .inc() } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 8510683c4..e7a4f0509 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -716,8 +716,4 @@ impl TrackedPipeline { Err(_) => Err(counts), } } - - pub fn size(&self) -> usize { - self.counts.values().sum() - } } diff --git a/prometheus-config.yaml b/prometheus-config.yaml new file mode 100644 index 000000000..848890eb6 --- /dev/null +++ b/prometheus-config.yaml @@ -0,0 +1,9 @@ +global: + scrape_interval: 1s + evaluation_interval: 5s + +scrape_configs: + - job_name: "prometheus" + honor_labels: true + static_configs: + - targets: ["host.docker.internal:8873", "host.docker.internal:8875"] From cde45947b8f9d392cb306e4b41c423c284049e0a Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 11 Oct 2024 14:25:05 +0200 Subject: [PATCH 50/72] delete and recreate the consumer (#162) --- grpc-ingest/src/config.rs | 9 ++ grpc-ingest/src/ingester.rs | 12 +- grpc-ingest/src/redis.rs | 235 ++++++++++++------------------------ 3 files changed, 95 insertions(+), 161 deletions(-) diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 65c28c658..23e31387e 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -56,6 +56,11 @@ pub struct ConfigIngestStream { deserialize_with = "deserialize_usize_str" )] pub max_concurrency: usize, + #[serde( + default = "ConfigIngestStream::default_ack_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub ack_concurrency: usize, #[serde( default = "ConfigIngestStream::default_xack_buffer_size", deserialize_with = "deserialize_usize_str" @@ -71,6 +76,10 @@ impl ConfigIngestStream { 2 } + pub const fn default_ack_concurrency() -> usize { + 5 + } + pub const fn default_xack_batch_max_idle() -> Duration { Duration::from_millis(10_000) } diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 000f747b4..939322a83 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -108,7 +108,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { .map_err(Into::into) }) }) - .start()?; + .start() + .await?; let account_stream = IngestStream::build() .config(config.accounts.clone()) .connection(connection.clone()) @@ -124,7 +125,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { .map_err(Into::into) }) }) - .start()?; + .start() + .await?; let transactions_stream = IngestStream::build() .config(config.transactions.clone()) .connection(connection.clone()) @@ -140,7 +142,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { .map_err(Into::into) }) }) - .start()?; + .start() + .await?; let snapshot_stream = IngestStream::build() .config(config.snapshots.clone()) .connection(connection.clone()) @@ -156,7 +159,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { .map_err(Into::into) }) }) - .start()?; + .start() + .await?; let mut shutdown = create_shutdown()?; diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index e7a4f0509..bf81c7545 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -12,10 +12,7 @@ use { program_transformers::{AccountInfo, TransactionInfo}, redis::{ aio::MultiplexedConnection, - streams::{ - StreamClaimOptions, StreamClaimReply, StreamId, StreamKey, StreamMaxlen, - StreamPendingCountReply, StreamReadOptions, StreamReadReply, - }, + streams::{StreamId, StreamKey, StreamMaxlen, StreamReadOptions, StreamReadReply}, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, solana_sdk::{pubkey::Pubkey, signature::Signature}, @@ -26,7 +23,7 @@ use { prelude::*, AsyncHandler, }, - tracing::{debug, error, info}, + tracing::{debug, error, warn}, yellowstone_grpc_proto::{ convert_from::{ create_message_instructions, create_meta_inner_instructions, create_pubkey_vec, @@ -41,7 +38,7 @@ pub enum IngestStreamJob { } pub struct IngestStreamStop { - shutdown_tx: tokio::sync::oneshot::Sender<()>, + shutdown_tx: tokio::sync::broadcast::Sender<()>, control: tokio::task::JoinHandle<()>, } @@ -340,43 +337,6 @@ impl IngestStream { self } - async fn pending( - &self, - connection: &mut MultiplexedConnection, - start: &str, - ) -> RedisResult> { - let config = Arc::clone(&self.config); - - let pending = redis::cmd("XPENDING") - .arg(&config.name) - .arg(&config.group) - .arg(start) - .arg("+") - .arg(config.batch_size) - .arg(&config.consumer) - .query_async::<_, StreamPendingCountReply>(connection) - .await?; - let ids: Vec<&str> = pending.ids.iter().map(|info| info.id.as_str()).collect(); - let opts = StreamClaimOptions::default(); - - let claimed: StreamClaimReply = connection - .xclaim_options( - &config.name, - &config.group, - &config.consumer, - 100, - &ids, - opts, - ) - .await?; - - if claimed.ids.is_empty() { - return Ok(None); - } - - Ok(Some(claimed)) - } - async fn read(&self, connection: &mut MultiplexedConnection) -> RedisResult { let config = &self.config; @@ -390,45 +350,47 @@ impl IngestStream { .await } - pub fn start(mut self) -> anyhow::Result { + pub async fn start(mut self) -> anyhow::Result { let config = Arc::clone(&self.config); - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let (shutdown_tx, _) = tokio::sync::broadcast::channel(1); let connection = self.connection.take().expect("Connection is required"); let group_create_connection = connection.clone(); let config_group_create = Arc::clone(&config); - tokio::task::spawn_blocking(move || { - let mut connection = group_create_connection.clone(); - let config = Arc::clone(&config_group_create); - - let rt = tokio::runtime::Runtime::new()?; - - rt.block_on(async { - if let Err(e) = xgroup_create( - &mut connection, - &config.name, - &config.group, - &config.consumer, - ) - .await - { - error!("redis=xgroup_create stream={} err={:?}", config.name, e); - } else { - debug!( - "redis=xgroup_create stream={} group={} consumer={}", - config.name, config.group, config.consumer - ); - } - }); + let mut connection = group_create_connection.clone(); + let config = Arc::clone(&config_group_create); + + // Delete the existing group + if let Err(e) = xgroup_delete(&mut connection, &config.name, &config.group).await { + error!("redis=xgroup_delete stream={} err={:?}", config.name, e); + } else { + debug!( + "redis=xgroup_delete stream={} group={}", + config.name, config.group + ); + } - Ok::<(), anyhow::Error>(()) - }); + // Recreate the group + if let Err(e) = xgroup_create( + &mut connection, + &config.name, + &config.group, + &config.consumer, + ) + .await + { + error!("redis=xgroup_create stream={} err={:?}", config.name, e); + } else { + debug!( + "redis=xgroup_create stream={} group={} consumer={}", + config.name, config.group, config.consumer + ); + } let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); - let (ack_shutdown_tx, ack_shutdown_rx) = tokio::sync::oneshot::channel(); let xack_executor = Executor::builder(Nonblock(Tokio)) .max_concurrency(Some(10)) @@ -440,9 +402,9 @@ impl IngestStream { let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); + let mut shutdown_rx = shutdown_tx.subscribe(); async move { - let mut shutdown_rx = ack_shutdown_rx; let deadline = tokio::time::sleep(config.xack_batch_max_idle); tokio::pin!(deadline); @@ -452,7 +414,6 @@ impl IngestStream { pending.push(id); let count = pending.len(); if count >= config.xack_batch_max_size { - let ids = std::mem::take(&mut pending); xack_executor.push(AcknowledgeJob::Submit(ids)); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); @@ -460,13 +421,15 @@ impl IngestStream { }, _ = &mut deadline, if !pending.is_empty() => { let ids = std::mem::take(&mut pending); - xack_executor.push(AcknowledgeJob::Submit(ids)); - deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); }, - _ = &mut shutdown_rx => { - xack_executor.join_async().await; + _ = shutdown_rx.recv() => { + if !pending.is_empty() { + let ids = std::mem::take(&mut pending); + xack_executor.push(AcknowledgeJob::Submit(ids)); + } + break; } } @@ -487,9 +450,7 @@ impl IngestStream { })?; let labels = vec![config.name.clone()]; - let (report_shutdown_tx, report_shutdown_rx) = tokio::sync::oneshot::channel(); - let report_thread = tokio::spawn({ - let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = report_shutdown_rx; + let report = tokio::spawn({ let connection = connection.clone(); let config = Arc::clone(&config); @@ -497,31 +458,23 @@ impl IngestStream { let config = Arc::clone(&config); loop { - tokio::select! { - _ = &mut shutdown_rx => { - debug!( - "redis=report_thread stream={} Shutdown signal received, exiting report loop", - config.name - ); - break; - }, - _ = sleep(Duration::from_millis(100)) => { - if let Err(e) = report_xlen(connection.clone(), labels.clone()).await { - error!("redis=report_xlen err={:?}", e); - } - - debug!( - "redis=report_thread stream={} msg=waiting for pending messages", - config.name - ); - }, + sleep(Duration::from_millis(100)).await; + if let Err(e) = report_xlen(connection.clone(), labels.clone()).await { + error!("redis=report_xlen err={:?}", e); } + + debug!( + "redis=report_thread stream={} msg=xlen metric updated", + config.name + ); } } }); let control = tokio::spawn({ let mut connection = connection.clone(); + let mut shutdown_rx = shutdown_tx.subscribe(); + async move { let config = Arc::clone(&config); @@ -530,50 +483,20 @@ impl IngestStream { config.name ); - let mut shutdown_rx: tokio::sync::oneshot::Receiver<()> = shutdown_rx; - - let mut start = "-".to_owned(); + debug!( + "redis=read_stream stream={} msg=starting read stream", + &config.name + ); loop { tokio::select! { - _ = &mut shutdown_rx => { - debug!( - "redis=read_stream stream={} Shutdown signal received, exiting loops", - config.name - ); - break; - }, - claimed = self.pending(&mut connection, &start) => { - if let Ok(Some(claimed)) = claimed { - - let ids = claimed.ids.clone(); - let ids: Vec<&str> = ids.iter().map(|info| info.id.as_str()).collect(); - - info!("redis=claimed stream={} claimed={:?}", config.name, ids.len()); - - - for StreamId { id, map } in claimed.ids.into_iter() { - executor.push(IngestStreamJob::Process((id, map))); - } + _ = shutdown_rx.recv() => { + report.abort(); - - if let Some(last) = ids.last() { - start = last.to_string(); - } - } else { - break; + if let Err(e) = ack.await { + error!("Error during ack shutdown: {:?}", e); } - }, - } - } - loop { - tokio::select! { - _ = &mut shutdown_rx => { - debug!( - "redis=read_stream stream={} Shutdown signal received, exiting read loop", - config.name - ); break; }, result = self.read(&mut connection) => { @@ -599,27 +522,7 @@ impl IngestStream { } } - debug!("stream={} msg=start shut down ingest stream", config.name); - - executor.join_async().await; - - if let Err(e) = report_shutdown_tx.send(()) { - error!("Failed to send report shutdown signal: {:?}", e); - } - - if let Err(e) = ack_shutdown_tx.send(()) { - error!("Failed to send ack shutdown signal: {:?}", e); - } - - if let Err(e) = ack.await { - error!("Error during ack shutdown: {:?}", e); - } - - if let Err(e) = report_thread.await { - error!("Error during report thread shutdown: {:?}", e); - } - - debug!("stream={} msg=shut down stream", config.name); + warn!("stream={} msg=stream shutdown", config.name); } }); @@ -675,6 +578,24 @@ pub async fn xgroup_create( Ok(()) } +pub async fn xgroup_delete( + connection: &mut C, + name: &str, + group: &str, +) -> anyhow::Result<()> { + let result: RedisResult = redis::cmd("XGROUP") + .arg("DESTROY") + .arg(name) + .arg(group) + .query_async(connection) + .await; + + match result { + Ok(_) => Ok(()), + Err(error) => Err(error.into()), + } +} + pub struct TrackedPipeline { pipeline: redis::Pipeline, counts: HashMap, From 0005b3ef2c531a860a0539fe7a62964c34ab1292 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Sat, 12 Oct 2024 15:28:34 +0200 Subject: [PATCH 51/72] Type out redis message handlers (#163) --- grpc-ingest/src/ingester.rs | 113 ++------ grpc-ingest/src/prom.rs | 12 + grpc-ingest/src/redis.rs | 518 +++++++++++++++++++++--------------- 3 files changed, 350 insertions(+), 293 deletions(-) diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 939322a83..64771ce2c 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -3,12 +3,12 @@ use { config::{ConfigIngester, REDIS_STREAM_DATA_KEY}, postgres::{create_pool as pg_create_pool, report_pgpool}, prom::redis_xack_inc, - redis::{IngestStream, RedisStreamMessage}, + redis::{AccountHandle, DownloadMetadataJsonHandle, IngestStream, TransactionHandle}, util::create_shutdown, }, das_core::{DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier}, futures::{future::BoxFuture, stream::StreamExt}, - program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, + program_transformers::ProgramTransformer, redis::aio::MultiplexedConnection, std::sync::Arc, tokio::time::{sleep, Duration}, @@ -60,112 +60,55 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { let download_metadata_stream = config.download_metadata.stream.clone(); let download_metadata_stream_maxlen = config.download_metadata.stream_maxlen; - let accounts_download_metadata_notifier = download_metadata_notifier_v2( - connection.clone(), - download_metadata_stream.name.clone(), - download_metadata_stream_maxlen, - )?; - let snapshots_download_metadata_notifier = download_metadata_notifier_v2( - connection.clone(), - download_metadata_stream.name.clone(), - download_metadata_stream_maxlen, - )?; - let transactions_download_metadata_notifier = download_metadata_notifier_v2( + let download_metadata_notifier = download_metadata_notifier_v2( connection.clone(), download_metadata_stream.name.clone(), download_metadata_stream_maxlen, )?; - let pt_accounts = Arc::new(ProgramTransformer::new( + let program_transformer = Arc::new(ProgramTransformer::new( pool.clone(), - accounts_download_metadata_notifier, - )); - let pt_snapshots = Arc::new(ProgramTransformer::new( - pool.clone(), - snapshots_download_metadata_notifier, - )); - let pt_transactions = Arc::new(ProgramTransformer::new( - pool.clone(), - transactions_download_metadata_notifier, + download_metadata_notifier, )); let http_client = reqwest::Client::builder() .timeout(config.download_metadata.request_timeout) .build()?; - let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); - let download_metadata_stream = IngestStream::build() + let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); + let download_metadatas = IngestStream::build() .config(config.download_metadata.stream.clone()) .connection(connection.clone()) - .handler(move |info| { - let download_metadata = Arc::clone(&download_metadata); - - Box::pin(async move { - let info = DownloadMetadataInfo::try_parse_msg(info)?; - - download_metadata - .handle_download(&info) - .await - .map_err(Into::into) - }) - }) + .handler(DownloadMetadataJsonHandle::new(Arc::clone( + &download_metadata, + ))) .start() .await?; - let account_stream = IngestStream::build() - .config(config.accounts.clone()) + + let accounts = IngestStream::build() + .config(config.accounts) .connection(connection.clone()) - .handler(move |info| { - let pt_accounts = Arc::clone(&pt_accounts); - - Box::pin(async move { - let info = AccountInfo::try_parse_msg(info)?; - - pt_accounts - .handle_account_update(&info) - .await - .map_err(Into::into) - }) - }) + .handler(AccountHandle::new(Arc::clone(&program_transformer))) .start() .await?; - let transactions_stream = IngestStream::build() - .config(config.transactions.clone()) + + let transactions = IngestStream::build() + .config(config.transactions) .connection(connection.clone()) - .handler(move |info| { - let pt_transactions = Arc::clone(&pt_transactions); - - Box::pin(async move { - let info = TransactionInfo::try_parse_msg(info)?; - - pt_transactions - .handle_transaction(&info) - .await - .map_err(Into::into) - }) - }) + .handler(TransactionHandle::new(Arc::clone(&program_transformer))) .start() .await?; - let snapshot_stream = IngestStream::build() - .config(config.snapshots.clone()) + + let snapshots = IngestStream::build() + .config(config.snapshots) .connection(connection.clone()) - .handler(move |info| { - let pt_snapshots = Arc::clone(&pt_snapshots); - - Box::pin(async move { - let info = AccountInfo::try_parse_msg(info)?; - - pt_snapshots - .handle_account_update(&info) - .await - .map_err(Into::into) - }) - }) + .handler(AccountHandle::new(Arc::clone(&program_transformer))) .start() .await?; let mut shutdown = create_shutdown()?; let report_pool = pool.clone(); - let report_handle = tokio::spawn(async move { + let report = tokio::spawn(async move { let pool = report_pool.clone(); loop { sleep(Duration::from_millis(100)).await; @@ -177,12 +120,12 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { warn!("{signal} received, waiting for spawned tasks..."); } - report_handle.abort(); + report.abort(); - account_stream.stop().await?; - transactions_stream.stop().await?; - download_metadata_stream.stop().await?; - snapshot_stream.stop().await?; + accounts.stop().await?; + transactions.stop().await?; + download_metadatas.stop().await?; + snapshots.stop().await?; pool.close().await; diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index aaa8ac273..91ce3e87b 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -30,6 +30,11 @@ lazy_static::lazy_static! { &["stream", "status"] ).unwrap(); + static ref REDIS_XREAD_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xread_count", "Count of messages seen"), + &["stream"] + ).unwrap(); + static ref REDIS_XACK_COUNT: IntCounterVec = IntCounterVec::new( Opts::new("redis_xack_count", "Total number of processed messages"), &["stream"] @@ -72,6 +77,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(VERSION_INFO_METRIC); register!(REDIS_STREAM_LENGTH); register!(REDIS_XADD_STATUS_COUNT); + register!(REDIS_XREAD_COUNT); register!(REDIS_XACK_COUNT); register!(PGPOOL_CONNECTIONS); register!(PROGRAM_TRANSFORMER_TASKS); @@ -141,6 +147,12 @@ pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) .inc_by(delta as u64); } +pub fn redis_xread_inc(stream: &str, delta: usize) { + REDIS_XREAD_COUNT + .with_label_values(&[stream]) + .inc_by(delta as u64) +} + pub fn redis_xack_inc(stream: &str, delta: usize) { REDIS_XACK_COUNT .with_label_values(&[stream]) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index bf81c7545..61a245a4a 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -3,20 +3,20 @@ use { config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, prom::{ ingest_tasks_reset, ingest_tasks_total_dec, ingest_tasks_total_inc, - program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, + program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, redis_xread_inc, ProgramTransformerTaskStatusKind, }, }, - das_core::DownloadMetadataInfo, + das_core::{DownloadMetadata, DownloadMetadataInfo}, futures::future::BoxFuture, - program_transformers::{AccountInfo, TransactionInfo}, + program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, redis::{ aio::MultiplexedConnection, streams::{StreamId, StreamKey, StreamMaxlen, StreamReadOptions, StreamReadReply}, AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, }, solana_sdk::{pubkey::Pubkey, signature::Signature}, - std::{collections::HashMap, sync::Arc}, + std::{collections::HashMap, marker::PhantomData, sync::Arc}, tokio::time::{sleep, Duration}, topograph::{ executor::{Executor, Nonblock, Tokio}, @@ -33,85 +33,6 @@ use { }, }; -pub enum IngestStreamJob { - Process((String, HashMap)), -} - -pub struct IngestStreamStop { - shutdown_tx: tokio::sync::broadcast::Sender<()>, - control: tokio::task::JoinHandle<()>, -} - -impl IngestStreamStop { - pub async fn stop(self) -> anyhow::Result<()> { - let _ = self.shutdown_tx.send(()); - - self.control.await?; - - Ok(()) - } -} - -type HandlerFn = dyn Fn(HashMap) -> BoxFuture<'static, Result<(), IngestMessageError>> - + Send - + Sync; - -#[derive(Clone)] -pub struct IngestStreamHandler { - handler: Arc, - ack_sender: tokio::sync::mpsc::Sender, - config: Arc, -} - -impl<'a> - AsyncHandler>> - for IngestStreamHandler -{ - type Output = (); - - fn handle( - &self, - job: IngestStreamJob, - _handle: topograph::executor::Handle<'a, IngestStreamJob, Nonblock>, - ) -> impl futures::Future + Send { - let handler = Arc::clone(&self.handler); - let ack_sender = self.ack_sender.clone(); - let config = Arc::clone(&self.config); - - ingest_tasks_total_inc(&config.name); - async move { - match job { - IngestStreamJob::Process((id, msg)) => { - match handler(msg).await { - Ok(()) => program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::Success, - ), - Err(IngestMessageError::RedisStreamMessage(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::DownloadMetadataJson(e)) => { - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::ProgramTransformer(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - } - - if let Err(e) = ack_sender.send(id).await { - error!("Failed to send ack id to channel: {:?}", e); - } - - ingest_tasks_total_dec(&config.name); - } - } - } - } -} - #[derive(thiserror::Error, Debug)] pub enum RedisStreamMessageError { #[error("failed to get data (key: {0}) from stream")] @@ -248,6 +169,208 @@ pub enum IngestMessageError { DownloadMetadataJson(#[from] das_core::MetadataJsonTaskError), } +pub enum IngestStreamJob { + Process((String, HashMap)), +} + +pub struct IngestStreamStop { + shutdown_tx: tokio::sync::oneshot::Sender<()>, + control: tokio::task::JoinHandle<()>, +} + +impl IngestStreamStop { + pub async fn stop(self) -> anyhow::Result<()> { + self.shutdown_tx + .send(()) + .map_err(|_| anyhow::anyhow!("Failed to send shutdown signal"))?; + + self.control.await?; + + Ok(()) + } +} + +pub trait MessageHandler: Send + Sync + Clone + 'static { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>>; +} + +pub struct DownloadMetadataJsonHandle(Arc); + +impl MessageHandler for DownloadMetadataJsonHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let download_metadata = Arc::clone(&self.0); + + Box::pin(async move { + let info = DownloadMetadataInfo::try_parse_msg(input)?; + download_metadata + .handle_download(&info) + .await + .map_err(Into::into) + }) + } +} + +impl DownloadMetadataJsonHandle { + pub fn new(download_metadata: Arc) -> Self { + Self(download_metadata) + } +} + +impl Clone for DownloadMetadataJsonHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +pub struct AccountHandle(Arc); + +impl AccountHandle { + pub fn new(program_transformer: Arc) -> Self { + Self(program_transformer) + } +} + +impl MessageHandler for AccountHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let program_transformer = Arc::clone(&self.0); + Box::pin(async move { + let account = AccountInfo::try_parse_msg(input)?; + program_transformer + .handle_account_update(&account) + .await + .map_err(IngestMessageError::ProgramTransformer) + }) + } +} + +impl Clone for AccountHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +pub struct TransactionHandle(Arc); + +impl TransactionHandle { + pub fn new(program_transformer: Arc) -> Self { + Self(program_transformer) + } +} + +impl MessageHandler for TransactionHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let program_transformer = Arc::clone(&self.0); + + Box::pin(async move { + let transaction = TransactionInfo::try_parse_msg(input)?; + program_transformer + .handle_transaction(&transaction) + .await + .map_err(IngestMessageError::ProgramTransformer) + }) + } +} + +impl Clone for TransactionHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +#[derive(Clone)] +pub struct IngestStreamHandler +where + H: MessageHandler + Clone + 'static, +{ + ack_sender: tokio::sync::mpsc::Sender, + config: Arc, + handler: H, + _marker: PhantomData, +} + +impl IngestStreamHandler +where + H: MessageHandler + Clone + 'static, +{ + pub fn new( + ack_sender: tokio::sync::mpsc::Sender, + config: Arc, + handler: H, + ) -> Self { + Self { + ack_sender, + config, + handler, + _marker: PhantomData, + } + } +} + +impl<'a, T> + AsyncHandler>> + for IngestStreamHandler +where + T: MessageHandler + Clone, +{ + type Output = (); + + fn handle( + &self, + job: IngestStreamJob, + _handle: topograph::executor::Handle<'a, IngestStreamJob, Nonblock>, + ) -> impl futures::Future + Send { + let handler = &self.handler; + let ack_sender = &self.ack_sender; + let config = &self.config; + + ingest_tasks_total_inc(&config.name); + async move { + match job { + IngestStreamJob::Process((id, msg)) => { + let result = handler.handle(msg).await.map_err(Into::into); + + match result { + Ok(()) => program_transformer_task_status_inc( + ProgramTransformerTaskStatusKind::Success, + ), + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + } + + if let Err(e) = ack_sender.send(id).await { + error!("Failed to send ack id to channel: {:?}", e); + } + + ingest_tasks_total_dec(&config.name); + } + } + } + } +} + #[derive(Clone, Debug)] enum AcknowledgeJob { Submit(Vec), @@ -287,35 +410,47 @@ impl<'a> { Ok(response) => { debug!( - "Acknowledged and deleted message: stream={:?} response={:?} expected={:?}", - &config.name, response, count + target: "acknowledge_handler", + "action=acknowledge_and_delete stream={} response={:?} expected={:?}", + config.name, response, count ); redis_xack_inc(&config.name, count); } Err(e) => { - error!("Failed to acknowledge or delete message: error={:?}", e); + error!( + target: "acknowledge_handler", + "action=acknowledge_and_delete_failed stream={} error={:?}", + config.name, e + ); } } } } } -pub struct IngestStream { +pub struct IngestStream { config: Arc, connection: Option, - handler: Option>, + handler: Option, + _handler: PhantomData, } -impl IngestStream { +impl IngestStream { pub fn build() -> Self { Self { config: Arc::new(ConfigIngestStream::default()), connection: None, handler: None, + _handler: PhantomData, } } + pub fn handler(mut self, handler: H) -> Self { + self.handler = Some(handler); + self + } + pub fn config(mut self, config: ConfigIngestStream) -> Self { self.config = Arc::new(config); self @@ -326,24 +461,13 @@ impl IngestStream { self } - pub fn handler(mut self, handler: F) -> Self - where - F: Fn(HashMap) -> BoxFuture<'static, Result<(), IngestMessageError>> - + Send - + Sync - + 'static, - { - self.handler = Some(Arc::new(handler)); - self - } - async fn read(&self, connection: &mut MultiplexedConnection) -> RedisResult { let config = &self.config; let opts = StreamReadOptions::default() .group(&config.group, &config.consumer) .count(config.batch_size) - .block(100); + .block(250); connection .xread_options(&[&config.name], &[">"], &opts) @@ -353,27 +477,17 @@ impl IngestStream { pub async fn start(mut self) -> anyhow::Result { let config = Arc::clone(&self.config); - let (shutdown_tx, _) = tokio::sync::broadcast::channel(1); - - let connection = self.connection.take().expect("Connection is required"); + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); - let group_create_connection = connection.clone(); - let config_group_create = Arc::clone(&config); - - let mut connection = group_create_connection.clone(); - let config = Arc::clone(&config_group_create); + let mut connection = self.connection.take().expect("Connection is required"); + let handler = self.handler.take().expect("Handler is required"); - // Delete the existing group if let Err(e) = xgroup_delete(&mut connection, &config.name, &config.group).await { - error!("redis=xgroup_delete stream={} err={:?}", config.name, e); + error!(target: "ingest_stream", "action=xgroup_delete stream={} error={:?}", config.name, e); } else { - debug!( - "redis=xgroup_delete stream={} group={}", - config.name, config.group - ); + debug!(target: "ingest_stream", "action=xgroup_delete stream={} group={}", config.name, config.group); } - // Recreate the group if let Err(e) = xgroup_create( &mut connection, &config.name, @@ -382,27 +496,24 @@ impl IngestStream { ) .await { - error!("redis=xgroup_create stream={} err={:?}", config.name, e); + error!(target: "ingest_stream", "action=xgroup_create stream={} error={:?}", config.name, e); } else { - debug!( - "redis=xgroup_create stream={} group={} consumer={}", - config.name, config.group, config.consumer - ); + debug!(target: "ingest_stream", "action=xgroup_create stream={} group={} consumer={}", config.name, config.group, config.consumer); } let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); - let xack_executor = Executor::builder(Nonblock(Tokio)) - .max_concurrency(Some(10)) + .max_concurrency(Some(config.ack_concurrency)) .build_async(AcknowledgeHandler { config: Arc::clone(&config), connection: connection.clone(), })?; + let (ack_shutdown_tx, mut ack_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); - let mut shutdown_rx = shutdown_tx.subscribe(); async move { let deadline = tokio::time::sleep(config.xack_batch_max_idle); @@ -412,42 +523,40 @@ impl IngestStream { tokio::select! { Some(id) = ack_rx.recv() => { pending.push(id); - let count = pending.len(); - if count >= config.xack_batch_max_size { + if pending.len() >= config.xack_batch_max_size { let ids = std::mem::take(&mut pending); xack_executor.push(AcknowledgeJob::Submit(ids)); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } - }, + } _ = &mut deadline, if !pending.is_empty() => { let ids = std::mem::take(&mut pending); xack_executor.push(AcknowledgeJob::Submit(ids)); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); - }, - _ = shutdown_rx.recv() => { - if !pending.is_empty() { - let ids = std::mem::take(&mut pending); - xack_executor.push(AcknowledgeJob::Submit(ids)); - } - + } + _ = &mut ack_shutdown_rx => { break; } } } + + if !pending.is_empty() { + xack_executor.push(AcknowledgeJob::Submit(std::mem::take(&mut pending))); + } + + xack_executor.join_async().await; } }); - let handler = self.handler.take().expect("Handler is required"); - ingest_tasks_reset(&config.name); let executor = Executor::builder(Nonblock(Tokio)) .max_concurrency(Some(config.max_concurrency)) - .build_async(IngestStreamHandler { + .build_async(IngestStreamHandler::new( + ack_tx.clone(), + Arc::clone(&config), handler, - ack_sender: ack_tx.clone(), - config: Arc::clone(&config), - })?; + ))?; let labels = vec![config.name.clone()]; let report = tokio::spawn({ @@ -460,41 +569,35 @@ impl IngestStream { loop { sleep(Duration::from_millis(100)).await; if let Err(e) = report_xlen(connection.clone(), labels.clone()).await { - error!("redis=report_xlen err={:?}", e); + error!(target: "ingest_stream", "action=report_xlen stream={} error={:?}", config.name, e); } - debug!( - "redis=report_thread stream={} msg=xlen metric updated", - config.name - ); + debug!(target: "ingest_stream", "action=report_thread stream={} xlen metric updated", config.name); } } }); let control = tokio::spawn({ let mut connection = connection.clone(); - let mut shutdown_rx = shutdown_tx.subscribe(); async move { let config = Arc::clone(&config); - debug!( - "redis=read_stream stream={} Starting read stream task", - config.name - ); - - debug!( - "redis=read_stream stream={} msg=starting read stream", - &config.name - ); + debug!(target: "ingest_stream", "action=read_stream_start stream={}", config.name); loop { tokio::select! { - _ = shutdown_rx.recv() => { + _ = &mut shutdown_rx => { report.abort(); + executor.join_async().await; + + if let Err(e) = ack_shutdown_tx.send(()) { + error!(target: "ingest_stream", "action=send_shutdown_signal stream={} error={:?}", config.name, e); + } + if let Err(e) = ack.await { - error!("Error during ack shutdown: {:?}", e); + error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", config.name, e); } break; @@ -502,27 +605,26 @@ impl IngestStream { result = self.read(&mut connection) => { match result { Ok(reply) => { - let count = reply.keys.len(); - debug!( - "redis=xread stream={:?} count={:?}", - &config.name, count - ); - for StreamKey { key: _, ids } in reply.keys { + let count = ids.len(); + debug!(target: "ingest_stream", "action=xread stream={} count={:?}", &config.name, count); + + redis_xread_inc(&config.name, count); + for StreamId { id, map } in ids { executor.push(IngestStreamJob::Process((id, map))); } } } Err(err) => { - error!("redis=xread stream={:?} err={:?}", &config.name, err); + error!(target: "ingest_stream", "action=xread stream={} error={:?}", &config.name, err); } } } } } - warn!("stream={} msg=stream shutdown", config.name); + warn!(target: "ingest_stream", "action=stream_shutdown stream={} stream shutdown", config.name); } }); @@ -533,6 +635,49 @@ impl IngestStream { } } +pub struct TrackedPipeline { + pipeline: redis::Pipeline, + counts: HashMap, +} + +impl Default for TrackedPipeline { + fn default() -> Self { + Self { + pipeline: redis::pipe(), + counts: HashMap::new(), + } + } +} + +type TrackedStreamCounts = HashMap; + +impl TrackedPipeline { + pub fn xadd_maxlen(&mut self, key: &str, maxlen: StreamMaxlen, id: F, field: V) + where + F: redis::ToRedisArgs, + V: redis::ToRedisArgs, + { + self.pipeline + .xadd_maxlen(key, maxlen, id, &[(REDIS_STREAM_DATA_KEY, field)]); + *self.counts.entry(key.to_string()).or_insert(0) += 1; + } + + pub async fn flush( + &mut self, + connection: &mut MultiplexedConnection, + ) -> Result { + let result: RedisResult = self.pipeline.atomic().query_async(connection).await; + let counts = self.counts.clone(); + self.counts.clear(); + self.pipeline.clear(); + + match result { + Ok(_) => Ok(counts), + Err(_) => Err(counts), + } + } +} + pub async fn report_xlen( mut connection: C, streams: Vec, @@ -595,46 +740,3 @@ pub async fn xgroup_delete( Err(error) => Err(error.into()), } } - -pub struct TrackedPipeline { - pipeline: redis::Pipeline, - counts: HashMap, -} - -impl Default for TrackedPipeline { - fn default() -> Self { - Self { - pipeline: redis::pipe(), - counts: HashMap::new(), - } - } -} - -type TrackedStreamCounts = HashMap; - -impl TrackedPipeline { - pub fn xadd_maxlen(&mut self, key: &str, maxlen: StreamMaxlen, id: F, field: V) - where - F: redis::ToRedisArgs, - V: redis::ToRedisArgs, - { - self.pipeline - .xadd_maxlen(key, maxlen, id, &[(REDIS_STREAM_DATA_KEY, field)]); - *self.counts.entry(key.to_string()).or_insert(0) += 1; - } - - pub async fn flush( - &mut self, - connection: &mut MultiplexedConnection, - ) -> Result { - let result: RedisResult = self.pipeline.atomic().query_async(connection).await; - let counts = self.counts.clone(); - self.counts.clear(); - self.pipeline.clear(); - - match result { - Ok(_) => Ok(counts), - Err(_) => Err(counts), - } - } -} From d092ad8a2546a221ad5ae7797bbaef6a034c02db Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Sun, 13 Oct 2024 08:40:13 +0200 Subject: [PATCH 52/72] skip reading new messages if the ack buffer is full. track running tasks. (#164) --- grpc-ingest/config-ingester.yml | 4 +++- grpc-ingest/src/grpc.rs | 24 +++++++++++++++++------- grpc-ingest/src/ingester.rs | 7 ++++--- grpc-ingest/src/prom.rs | 28 ++++++++++++++++++++++++++-- grpc-ingest/src/redis.rs | 13 +++++++------ 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 02c6de877..9ca48eb15 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -3,19 +3,21 @@ redis: redis://localhost:6379 postgres: url: postgres://solana:solana@localhost/solana min_connections: 10 - max_connections: 50 # `max_connection` should be bigger than `program_transformer.max_tasks_in_process` otherwise unresolved lock is possible + max_connections: 50 snapshots: name: SNAPSHOTS max_concurrency: 10 batch_size: 100 xack_batch_max_idle_ms: 1_000 xack_buffer_size: 10_000 + xack_batch_max_size: 500 accounts: name: ACCOUNTS max_concurrency: 10 batch_size: 100 xack_batch_max_idle_ms: 1_000 xack_buffer_size: 10_000 + xack_batch_max_size: 500 transactions: name: TRANSACTIONS download_metadata: diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 6529f5194..606f2f096 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -1,6 +1,8 @@ use { crate::{ - config::ConfigGrpc, prom::redis_xadd_status_inc, redis::TrackedPipeline, + config::ConfigGrpc, + prom::{grpc_tasks_total_dec, grpc_tasks_total_inc, redis_xadd_status_inc}, + redis::TrackedPipeline, util::create_shutdown, }, anyhow::Context, @@ -56,6 +58,8 @@ impl<'a> AsyncHandler { @@ -68,7 +72,7 @@ impl<'a> AsyncHandler AsyncHandler { pipe.xadd_maxlen( @@ -99,6 +104,7 @@ impl<'a> AsyncHandler { let ping = subscribe_tx @@ -112,25 +118,29 @@ impl<'a> AsyncHandler { - debug!(message = "Ping sent successfully", id = PING_ID) + debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID) } Err(err) => { - warn!(message = "Failed to send ping", ?err, id = PING_ID) + warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID) } } } UpdateOneof::Pong(pong) => { if pong.id == PING_ID { - debug!(message = "Pong received", id = PING_ID); + debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); } else { - warn!(message = "Unknown pong id received", id = pong.id); + warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); } } - var => warn!(message = "Unknown update variant", ?var), + var => { + warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant", ?var) + } } } } } + + grpc_tasks_total_dec(); } } } diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 64771ce2c..03c08892d 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -122,10 +122,11 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { report.abort(); - accounts.stop().await?; - transactions.stop().await?; + futures::future::join_all(vec![accounts.stop(), transactions.stop(), snapshots.stop()]) + .await + .into_iter() + .collect::, _>>()?; download_metadatas.stop().await?; - snapshots.stop().await?; pool.close().await; diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 91ce3e87b..e70e14671 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -62,6 +62,16 @@ lazy_static::lazy_static! { Opts::new("ingest_tasks", "Number of tasks spawned for ingest"), &["stream"] ).unwrap(); + + static ref ACK_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("ack_tasks", "Number of tasks spawned for ack redis messages"), + &["stream"] + ).unwrap(); + + static ref GRPC_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("grpc_tasks", "Number of tasks spawned for writing grpc messages to redis "), + &[] + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -84,6 +94,8 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(PROGRAM_TRANSFORMER_TASK_STATUS_COUNT); register!(DOWNLOAD_METADATA_INSERTED_COUNT); register!(INGEST_TASKS); + register!(ACK_TASKS); + register!(GRPC_TASKS); VERSION_INFO_METRIC .with_label_values(&[ @@ -182,8 +194,20 @@ pub fn ingest_tasks_total_dec(stream: &str) { INGEST_TASKS.with_label_values(&[stream]).dec() } -pub fn ingest_tasks_reset(stream: &str) { - INGEST_TASKS.with_label_values(&[stream]).set(0) +pub fn ack_tasks_total_inc(stream: &str) { + ACK_TASKS.with_label_values(&[stream]).inc() +} + +pub fn ack_tasks_total_dec(stream: &str) { + ACK_TASKS.with_label_values(&[stream]).dec() +} + +pub fn grpc_tasks_total_inc() { + GRPC_TASKS.with_label_values(&[]).inc() +} + +pub fn grpc_tasks_total_dec() { + GRPC_TASKS.with_label_values(&[]).dec() } #[derive(Debug, Clone, Copy)] diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 61a245a4a..42229be34 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -2,9 +2,9 @@ use { crate::{ config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, prom::{ - ingest_tasks_reset, ingest_tasks_total_dec, ingest_tasks_total_inc, - program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, redis_xread_inc, - ProgramTransformerTaskStatusKind, + ack_tasks_total_dec, ack_tasks_total_inc, ingest_tasks_total_dec, + ingest_tasks_total_inc, program_transformer_task_status_inc, redis_xack_inc, + redis_xlen_set, redis_xread_inc, ProgramTransformerTaskStatusKind, }, }, das_core::{DownloadMetadata, DownloadMetadataInfo}, @@ -400,6 +400,7 @@ impl<'a> let count = ids.len(); + ack_tasks_total_inc(&config.name); async move { match redis::pipe() .atomic() @@ -425,6 +426,8 @@ impl<'a> ); } } + + ack_tasks_total_dec(&config.name); } } } @@ -548,8 +551,6 @@ impl IngestStream { } }); - ingest_tasks_reset(&config.name); - let executor = Executor::builder(Nonblock(Tokio)) .max_concurrency(Some(config.max_concurrency)) .build_async(IngestStreamHandler::new( @@ -602,7 +603,7 @@ impl IngestStream { break; }, - result = self.read(&mut connection) => { + result = self.read(&mut connection), if ack_tx.capacity() >= config.batch_size => { match result { Ok(reply) => { for StreamKey { key: _, ids } in reply.keys { From 6f158f38fc0ac7c16b698f146f48b9693a9187fe Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 14 Oct 2024 10:57:57 +0200 Subject: [PATCH 53/72] only push handler job for ingest if capacity (#165) --- grpc-ingest/config-ingester.yml | 2 +- grpc-ingest/src/grpc.rs | 75 ++++++++++++++++----------------- grpc-ingest/src/ingester.rs | 34 ++++++++++++--- grpc-ingest/src/redis.rs | 68 +++++++++++++++++------------- 4 files changed, 104 insertions(+), 75 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index 9ca48eb15..a18ae2a54 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -21,6 +21,6 @@ accounts: transactions: name: TRANSACTIONS download_metadata: - max_attempts: 3 + max_attempts: 1 stream: name: METADATA_JSON diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 606f2f096..f1a84b9d1 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -6,9 +6,9 @@ use { util::create_shutdown, }, anyhow::Context, - futures::{channel::mpsc::SendError, stream::StreamExt, Sink, SinkExt}, + futures::{stream::StreamExt, SinkExt}, redis::streams::StreamMaxlen, - std::{collections::HashMap, pin::Pin, sync::Arc, time::Duration}, + std::{collections::HashMap, sync::Arc, time::Duration}, tokio::{sync::Mutex, time::sleep}, topograph::{ executor::{Executor, Nonblock, Tokio}, @@ -32,14 +32,11 @@ enum GrpcJob { ProcessSubscribeUpdate(Box), } -type SubscribeTx = Pin + Send + Sync>>; - #[derive(Clone)] pub struct GrpcJobHandler { connection: redis::aio::MultiplexedConnection, config: Arc, pipe: Arc>, - subscribe_tx: Arc>, } impl<'a> AsyncHandler>> @@ -56,8 +53,6 @@ impl<'a> AsyncHandler AsyncHandler { - let ping = subscribe_tx - .lock() - .await - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: PING_ID }), - ..Default::default() - }) - .await; - - match ping { - Ok(_) => { - debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID) - } - Err(err) => { - warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID) - } - } - } - UpdateOneof::Pong(pong) => { - if pong.id == PING_ID { - debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); - } else { - warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); - } - } - var => { - warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant", ?var) + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant") } } } @@ -178,7 +147,7 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { .await .context("failed to connect to gRPC")?; - let (subscribe_tx, stream) = dragon_mouth_client + let (mut subscribe_tx, stream) = dragon_mouth_client .subscribe_with_request(Some(request)) .await?; @@ -190,7 +159,6 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { config: Arc::clone(&config), connection: connection.clone(), pipe: Arc::clone(&pipe), - subscribe_tx: Arc::new(Mutex::new(Box::pin(subscribe_tx))), })?; let deadline_config = Arc::clone(&config); @@ -201,7 +169,38 @@ pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { exec.push(GrpcJob::FlushRedisPipe); } Some(Ok(msg)) = stream.next() => { - exec.push(GrpcJob::ProcessSubscribeUpdate(Box::new(msg))); + match msg.update_oneof { + Some(UpdateOneof::Account(_)) | Some(UpdateOneof::Transaction(_)) => { + exec.push(GrpcJob::ProcessSubscribeUpdate(Box::new(msg))); + } + Some(UpdateOneof::Ping(_)) => { + let ping = subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: PING_ID }), + ..Default::default() + }) + .await; + + match ping { + Ok(_) => { + debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID) + } + Err(err) => { + warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID) + } + } + } + Some(UpdateOneof::Pong(pong)) => { + if pong.id == PING_ID { + debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); + } else { + warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); + } + } + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant", ?msg.update_oneof) + } + } } _ = shutdown.next() => { exec.push(GrpcJob::FlushRedisPipe); diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 03c08892d..1dbdb42e5 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -12,7 +12,7 @@ use { redis::aio::MultiplexedConnection, std::sync::Arc, tokio::time::{sleep, Duration}, - tracing::warn, + tracing::{error, warn}, }; fn download_metadata_notifier_v2( @@ -117,16 +117,36 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { }); if let Some(signal) = shutdown.next().await { - warn!("{signal} received, waiting for spawned tasks..."); + warn!( + target: "ingester", + action = "shutdown_signal_received", + message = "Shutdown signal received, waiting for spawned tasks to complete", + signal = ?signal + ); } report.abort(); - futures::future::join_all(vec![accounts.stop(), transactions.stop(), snapshots.stop()]) - .await - .into_iter() - .collect::, _>>()?; - download_metadatas.stop().await?; + let (accounts, transactions, snapshots, download_metadatas) = futures::future::join4( + accounts.stop(), + transactions.stop(), + snapshots.stop(), + download_metadatas.stop(), + ) + .await; + + if let Err(e) = accounts { + error!(target: "ingester", action = "stop_accounts", message = "Failed to stop accounts stream cleanly", error = ?e); + } + if let Err(e) = transactions { + error!(target: "ingester", action = "stop_transactions", message = "Failed to stop transactions stream cleanly", error = ?e); + } + if let Err(e) = snapshots { + error!(target: "ingester", action = "stop_snapshots", message = "Failed to stop snapshots stream cleanly", error = ?e); + } + if let Err(e) = download_metadatas { + error!(target: "ingester", action = "stop_download_metadatas", message = "Failed to stop download_metadatas stream cleanly", error = ?e); + } pool.close().await; diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 42229be34..614067eec 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -17,7 +17,10 @@ use { }, solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{collections::HashMap, marker::PhantomData, sync::Arc}, - tokio::time::{sleep, Duration}, + tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, + time::{sleep, Duration}, + }, topograph::{ executor::{Executor, Nonblock, Tokio}, prelude::*, @@ -170,7 +173,7 @@ pub enum IngestMessageError { } pub enum IngestStreamJob { - Process((String, HashMap)), + Process((String, HashMap, OwnedSemaphorePermit)), } pub struct IngestStreamStop { @@ -336,37 +339,37 @@ where let config = &self.config; ingest_tasks_total_inc(&config.name); - async move { - match job { - IngestStreamJob::Process((id, msg)) => { - let result = handler.handle(msg).await.map_err(Into::into); - - match result { - Ok(()) => program_transformer_task_status_inc( - ProgramTransformerTaskStatusKind::Success, - ), - Err(IngestMessageError::RedisStreamMessage(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::DownloadMetadataJson(e)) => { - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::ProgramTransformer(e)) => { - error!("Failed to process message: {:?}", e); - program_transformer_task_status_inc(e.into()); - } - } + async move { + let (id, msg, _permit) = match job { + IngestStreamJob::Process((id, msg, permit)) => (id, msg, permit), + }; + let result = handler.handle(msg).await.map_err(Into::into); + + match result { + Ok(()) => { + program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success) + } + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); - if let Err(e) = ack_sender.send(id).await { - error!("Failed to send ack id to channel: {:?}", e); - } + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); - ingest_tasks_total_dec(&config.name); + program_transformer_task_status_inc(e.into()); } } + + if let Err(e) = ack_sender.send(id).await { + error!("Failed to send ack id to channel: {:?}", e); + } + + ingest_tasks_total_dec(&config.name); } } } @@ -479,6 +482,7 @@ impl IngestStream { pub async fn start(mut self) -> anyhow::Result { let config = Arc::clone(&self.config); + let semaphore = Arc::new(Semaphore::new(config.max_concurrency)); let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); @@ -613,7 +617,13 @@ impl IngestStream { redis_xread_inc(&config.name, count); for StreamId { id, map } in ids { - executor.push(IngestStreamJob::Process((id, map))); + let semaphore = Arc::clone(&semaphore); + + if let Ok(permit) = semaphore.acquire_owned().await { + executor.push(IngestStreamJob::Process((id, map, permit))); + } else { + error!(target: "ingest_stream", "action=acquire_semaphore stream={} error=failed to acquire semaphore", config.name); + } } } } From a347066809368207c127c3edd3765e262df5b423 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 14 Oct 2024 14:09:40 +0200 Subject: [PATCH 54/72] Only throttle based on the task semaphore (#166) --- grpc-ingest/src/ingester.rs | 31 ++++++++++--------------------- grpc-ingest/src/redis.rs | 2 +- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 1dbdb42e5..a3d23d029 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -2,7 +2,7 @@ use { crate::{ config::{ConfigIngester, REDIS_STREAM_DATA_KEY}, postgres::{create_pool as pg_create_pool, report_pgpool}, - prom::redis_xack_inc, + prom::{redis_xadd_status_inc}, redis::{AccountHandle, DownloadMetadataJsonHandle, IngestStream, TransactionHandle}, util::create_shutdown, }, @@ -12,7 +12,7 @@ use { redis::aio::MultiplexedConnection, std::sync::Arc, tokio::time::{sleep, Duration}, - tracing::{error, warn}, + tracing::{warn}, }; fn download_metadata_notifier_v2( @@ -32,7 +32,7 @@ fn download_metadata_notifier_v2( let info_bytes = serde_json::to_vec(&info)?; - redis::cmd("XADD") + let xadd = redis::cmd("XADD") .arg(&stream) .arg("MAXLEN") .arg("~") @@ -40,10 +40,12 @@ fn download_metadata_notifier_v2( .arg("*") .arg(REDIS_STREAM_DATA_KEY) .arg(info_bytes) - .query_async(&mut connection) - .await?; + .query_async::<_, redis::Value>(&mut connection) + .await; - redis_xack_inc(&stream, 1); + let status = xadd.map(|_| ()).map_err(|_| ()); + + redis_xadd_status_inc(&stream, status, 1); Ok(()) }) @@ -127,27 +129,14 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { report.abort(); - let (accounts, transactions, snapshots, download_metadatas) = futures::future::join4( + futures::future::join_all(vec![ accounts.stop(), transactions.stop(), snapshots.stop(), download_metadatas.stop(), - ) + ]) .await; - if let Err(e) = accounts { - error!(target: "ingester", action = "stop_accounts", message = "Failed to stop accounts stream cleanly", error = ?e); - } - if let Err(e) = transactions { - error!(target: "ingester", action = "stop_transactions", message = "Failed to stop transactions stream cleanly", error = ?e); - } - if let Err(e) = snapshots { - error!(target: "ingester", action = "stop_snapshots", message = "Failed to stop snapshots stream cleanly", error = ?e); - } - if let Err(e) = download_metadatas { - error!(target: "ingester", action = "stop_download_metadatas", message = "Failed to stop download_metadatas stream cleanly", error = ?e); - } - pool.close().await; Ok::<(), anyhow::Error>(()) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 614067eec..58aab463e 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -607,7 +607,7 @@ impl IngestStream { break; }, - result = self.read(&mut connection), if ack_tx.capacity() >= config.batch_size => { + result = self.read(&mut connection) => { match result { Ok(reply) => { for StreamKey { key: _, ids } in reply.keys { From 188c8780033956066340955c6a11d414966f50b2 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 14 Oct 2024 16:55:04 +0200 Subject: [PATCH 55/72] Remove topograph from ingest command (#167) --- grpc-ingest/src/ingester.rs | 8 +- grpc-ingest/src/redis.rs | 286 ++++++++++++++---------------------- 2 files changed, 113 insertions(+), 181 deletions(-) diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index a3d23d029..bc46b79f0 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -2,7 +2,7 @@ use { crate::{ config::{ConfigIngester, REDIS_STREAM_DATA_KEY}, postgres::{create_pool as pg_create_pool, report_pgpool}, - prom::{redis_xadd_status_inc}, + prom::redis_xadd_status_inc, redis::{AccountHandle, DownloadMetadataJsonHandle, IngestStream, TransactionHandle}, util::create_shutdown, }, @@ -12,7 +12,7 @@ use { redis::aio::MultiplexedConnection, std::sync::Arc, tokio::time::{sleep, Duration}, - tracing::{warn}, + tracing::warn, }; fn download_metadata_notifier_v2( @@ -127,8 +127,6 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { ); } - report.abort(); - futures::future::join_all(vec![ accounts.stop(), transactions.stop(), @@ -137,6 +135,8 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { ]) .await; + report.abort(); + pool.close().await; Ok::<(), anyhow::Error>(()) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 58aab463e..1e9d05783 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -8,7 +8,7 @@ use { }, }, das_core::{DownloadMetadata, DownloadMetadataInfo}, - futures::future::BoxFuture, + futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}, program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, redis::{ aio::MultiplexedConnection, @@ -17,15 +17,7 @@ use { }, solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{collections::HashMap, marker::PhantomData, sync::Arc}, - tokio::{ - sync::{OwnedSemaphorePermit, Semaphore}, - time::{sleep, Duration}, - }, - topograph::{ - executor::{Executor, Nonblock, Tokio}, - prelude::*, - AsyncHandler, - }, + tokio::time::{sleep, Duration}, tracing::{debug, error, warn}, yellowstone_grpc_proto::{ convert_from::{ @@ -172,10 +164,6 @@ pub enum IngestMessageError { DownloadMetadataJson(#[from] das_core::MetadataJsonTaskError), } -pub enum IngestStreamJob { - Process((String, HashMap, OwnedSemaphorePermit)), -} - pub struct IngestStreamStop { shutdown_tx: tokio::sync::oneshot::Sender<()>, control: tokio::task::JoinHandle<()>, @@ -293,145 +281,51 @@ impl Clone for TransactionHandle { } #[derive(Clone)] -pub struct IngestStreamHandler -where - H: MessageHandler + Clone + 'static, -{ - ack_sender: tokio::sync::mpsc::Sender, +pub struct Acknowledge { config: Arc, - handler: H, - _marker: PhantomData, -} - -impl IngestStreamHandler -where - H: MessageHandler + Clone + 'static, -{ - pub fn new( - ack_sender: tokio::sync::mpsc::Sender, - config: Arc, - handler: H, - ) -> Self { - Self { - ack_sender, - config, - handler, - _marker: PhantomData, - } - } + connection: MultiplexedConnection, } -impl<'a, T> - AsyncHandler>> - for IngestStreamHandler -where - T: MessageHandler + Clone, -{ - type Output = (); - - fn handle( - &self, - job: IngestStreamJob, - _handle: topograph::executor::Handle<'a, IngestStreamJob, Nonblock>, - ) -> impl futures::Future + Send { - let handler = &self.handler; - let ack_sender = &self.ack_sender; - let config = &self.config; - - ingest_tasks_total_inc(&config.name); - - async move { - let (id, msg, _permit) = match job { - IngestStreamJob::Process((id, msg, permit)) => (id, msg, permit), - }; - let result = handler.handle(msg).await.map_err(Into::into); - - match result { - Ok(()) => { - program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success) - } - Err(IngestMessageError::RedisStreamMessage(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::DownloadMetadataJson(e)) => { - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::ProgramTransformer(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - } - - if let Err(e) = ack_sender.send(id).await { - error!("Failed to send ack id to channel: {:?}", e); - } - - ingest_tasks_total_dec(&config.name); - } +impl Acknowledge { + pub fn new(config: Arc, connection: MultiplexedConnection) -> Self { + Self { config, connection } } } -#[derive(Clone, Debug)] -enum AcknowledgeJob { - Submit(Vec), -} - -#[derive(Clone)] -pub struct AcknowledgeHandler { - config: Arc, - connection: MultiplexedConnection, -} - -impl<'a> - AsyncHandler>> - for AcknowledgeHandler -{ - type Output = (); - - fn handle( - &self, - job: AcknowledgeJob, - _handle: topograph::executor::Handle<'a, AcknowledgeJob, Nonblock>, - ) -> impl futures::Future + Send { +impl Acknowledge { + async fn handle(&self, ids: Vec) { let mut connection = self.connection.clone(); - let config = Arc::clone(&self.config); - - let AcknowledgeJob::Submit(ids) = job; + let config = &self.config; let count = ids.len(); ack_tasks_total_inc(&config.name); - async move { - match redis::pipe() - .atomic() - .xack(&config.name, &config.group, &ids) - .xdel(&config.name, &ids) - .query_async::<_, redis::Value>(&mut connection) - .await - { - Ok(response) => { - debug!( - target: "acknowledge_handler", - "action=acknowledge_and_delete stream={} response={:?} expected={:?}", - config.name, response, count - ); - - redis_xack_inc(&config.name, count); - } - Err(e) => { - error!( - target: "acknowledge_handler", - "action=acknowledge_and_delete_failed stream={} error={:?}", - config.name, e - ); - } + match redis::pipe() + .atomic() + .xack(&config.name, &config.group, &ids) + .xdel(&config.name, &ids) + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(response) => { + debug!( + target: "acknowledge_handler", + "action=acknowledge_and_delete stream={} response={:?} expected={:?}", + config.name, response, count + ); + + redis_xack_inc(&config.name, count); + } + Err(e) => { + error!( + target: "acknowledge_handler", + "action=acknowledge_and_delete_failed stream={} error={:?}", + config.name, e + ); } - - ack_tasks_total_dec(&config.name); } + + ack_tasks_total_dec(&config.name); } } @@ -482,8 +376,6 @@ impl IngestStream { pub async fn start(mut self) -> anyhow::Result { let config = Arc::clone(&self.config); - let semaphore = Arc::new(Semaphore::new(config.max_concurrency)); - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); let mut connection = self.connection.take().expect("Connection is required"); @@ -509,18 +401,13 @@ impl IngestStream { } let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); - let xack_executor = Executor::builder(Nonblock(Tokio)) - .max_concurrency(Some(config.ack_concurrency)) - .build_async(AcknowledgeHandler { - config: Arc::clone(&config), - connection: connection.clone(), - })?; - let (ack_shutdown_tx, mut ack_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); + let mut tasks = FuturesUnordered::new(); + let handler = Arc::new(Acknowledge::new(Arc::clone(&config), connection.clone())); async move { let deadline = tokio::time::sleep(config.xack_batch_max_idle); @@ -530,15 +417,32 @@ impl IngestStream { tokio::select! { Some(id) = ack_rx.recv() => { pending.push(id); + if pending.len() >= config.xack_batch_max_size { + if tasks.len() >= config.ack_concurrency { + tasks.next().await; + } + let ids = std::mem::take(&mut pending); - xack_executor.push(AcknowledgeJob::Submit(ids)); + let handler = Arc::clone(&handler); + + tasks.push(tokio::spawn(async move { + handler.handle(ids).await; + })); + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } } _ = &mut deadline, if !pending.is_empty() => { + if tasks.len() >= config.ack_concurrency { + tasks.next().await; + } let ids = std::mem::take(&mut pending); - xack_executor.push(AcknowledgeJob::Submit(ids)); + let handler = Arc::clone(&handler); + tasks.push(tokio::spawn(async move { + handler.handle(ids).await; + })); + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } _ = &mut ack_shutdown_rx => { @@ -548,23 +452,16 @@ impl IngestStream { } if !pending.is_empty() { - xack_executor.push(AcknowledgeJob::Submit(std::mem::take(&mut pending))); + let handler = Arc::clone(&handler); + handler.handle(std::mem::take(&mut pending)).await; } - xack_executor.join_async().await; + while (tasks.next().await).is_some() {} } }); - let executor = Executor::builder(Nonblock(Tokio)) - .max_concurrency(Some(config.max_concurrency)) - .build_async(IngestStreamHandler::new( - ack_tx.clone(), - Arc::clone(&config), - handler, - ))?; - let labels = vec![config.name.clone()]; - let report = tokio::spawn({ + tokio::spawn({ let connection = connection.clone(); let config = Arc::clone(&config); @@ -573,36 +470,39 @@ impl IngestStream { loop { sleep(Duration::from_millis(100)).await; - if let Err(e) = report_xlen(connection.clone(), labels.clone()).await { - error!(target: "ingest_stream", "action=report_xlen stream={} error={:?}", config.name, e); - } + let connection = connection.clone(); + let labels = labels.clone(); - debug!(target: "ingest_stream", "action=report_thread stream={} xlen metric updated", config.name); + if let Err(e) = report_xlen(connection, labels).await { + error!(target: "ingest_stream", "action=report_xlen stream={} error={:?}", &config.name, e); + } } } }); let control = tokio::spawn({ let mut connection = connection.clone(); + let mut tasks = FuturesUnordered::new(); async move { let config = Arc::clone(&config); + let handler = handler.clone(); debug!(target: "ingest_stream", "action=read_stream_start stream={}", config.name); loop { + let config = Arc::clone(&config); + tokio::select! { _ = &mut shutdown_rx => { - report.abort(); - - executor.join_async().await; + while (tasks.next().await).is_some() {} if let Err(e) = ack_shutdown_tx.send(()) { - error!(target: "ingest_stream", "action=send_shutdown_signal stream={} error={:?}", config.name, e); + error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", &config.name, e); } if let Err(e) = ack.await { - error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", config.name, e); + error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", &config.name, e); } break; @@ -611,19 +511,51 @@ impl IngestStream { match result { Ok(reply) => { for StreamKey { key: _, ids } in reply.keys { + let config = Arc::clone(&config); let count = ids.len(); debug!(target: "ingest_stream", "action=xread stream={} count={:?}", &config.name, count); redis_xread_inc(&config.name, count); for StreamId { id, map } in ids { - let semaphore = Arc::clone(&semaphore); - - if let Ok(permit) = semaphore.acquire_owned().await { - executor.push(IngestStreamJob::Process((id, map, permit))); - } else { - error!(target: "ingest_stream", "action=acquire_semaphore stream={} error=failed to acquire semaphore", config.name); + if tasks.len() >= config.max_concurrency { + tasks.next().await; } + + let handler = handler.clone(); + let ack_tx = ack_tx.clone(); + let config = Arc::clone(&config); + + ingest_tasks_total_inc(&config.name); + + tasks.push(tokio::spawn(async move { + let result = handler.handle(map).await.map_err(IngestMessageError::into); + + match result { + Ok(()) => { + program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success); + } + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); + + program_transformer_task_status_inc(e.into()); + } + } + + ingest_tasks_total_dec(&config.name); + + if let Err(e) = ack_tx.send(id).await { + error!(target: "ingest_stream", "action=send_ack stream={} error={:?}", &config.name, e); + } + })); } } } From e15172f7b36d239f9a39495237003641598a2b8c Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 21 Oct 2024 17:23:58 +0200 Subject: [PATCH 56/72] Configure database connection idle timeout and max lifetime (#169) --- grpc-ingest/src/config.rs | 18 ++++++++++++++++++ grpc-ingest/src/postgres.rs | 2 ++ 2 files changed, 20 insertions(+) diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 23e31387e..64ef42953 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -243,6 +243,16 @@ pub struct ConfigIngesterPostgres { deserialize_with = "deserialize_usize_str" )] pub max_connections: usize, + #[serde( + default = "ConfigIngesterPostgres::default_idle_timeout", + deserialize_with = "deserialize_duration_str" + )] + pub idle_timeout: Duration, + #[serde( + default = "ConfigIngesterPostgres::default_max_lifetime", + deserialize_with = "deserialize_duration_str" + )] + pub max_lifetime: Duration, } impl ConfigIngesterPostgres { @@ -253,6 +263,14 @@ impl ConfigIngesterPostgres { pub const fn default_max_connections() -> usize { 50 } + + pub const fn default_idle_timeout() -> Duration { + Duration::from_millis(75) + } + + pub const fn default_max_lifetime() -> Duration { + Duration::from_millis(125) + } } #[derive(Debug, Clone, Default, Deserialize)] diff --git a/grpc-ingest/src/postgres.rs b/grpc-ingest/src/postgres.rs index 07e0ff929..1077bc49b 100644 --- a/grpc-ingest/src/postgres.rs +++ b/grpc-ingest/src/postgres.rs @@ -14,6 +14,8 @@ pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result Date: Tue, 22 Oct 2024 23:28:30 +0530 Subject: [PATCH 57/72] Add single NFT fetch and backfill feature to ops (#170) * feat: Add NFT single backfill command (ops) to fetch mint,tokenAcc,metadata for a nft and backfill it to the db * chore: cleanup * chore: optimize --- Cargo.lock | 2 + core/src/solana_rpc.rs | 27 +++++++++++- ops/Cargo.toml | 2 + ops/src/account/cmd.rs | 8 +++- ops/src/account/mod.rs | 1 + ops/src/account/nft.rs | 95 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 ops/src/account/nft.rs diff --git a/Cargo.lock b/Cargo.lock index 0f6b76380..856a5c9a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1945,8 +1945,10 @@ dependencies = [ "indicatif", "log", "mpl-bubblegum", + "mpl-token-metadata", "program_transformers", "sea-orm", + "serde_json", "solana-account-decoder", "solana-client", "solana-program", diff --git a/core/src/solana_rpc.rs b/core/src/solana_rpc.rs index b517f1404..26e9e1f05 100644 --- a/core/src/solana_rpc.rs +++ b/core/src/solana_rpc.rs @@ -3,13 +3,16 @@ use backon::ExponentialBuilder; use backon::Retryable; use clap::Parser; use solana_account_decoder::UiAccountEncoding; -use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; +use solana_client::rpc_response::RpcTokenAccountBalance; use solana_client::{ client_error::ClientError, nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcTransactionConfig}, rpc_filter::RpcFilterType, + rpc_request::RpcRequest, + rpc_response::Response as RpcResponse, + rpc_response::RpcConfirmedTransactionStatusWithSignature, }; use solana_sdk::{ account::Account, @@ -157,4 +160,26 @@ impl Rpc { .await? .value) } + + pub async fn get_token_largest_account(&self, mint: Pubkey) -> anyhow::Result { + Ok((|| async { + self.0 + .send::>>( + RpcRequest::Custom { + method: "getTokenLargestAccounts", + }, + serde_json::json!([mint.to_string(),]), + ) + .await + }) + .retry(&ExponentialBuilder::default()) + .await? + .value + .first() + .ok_or(anyhow::anyhow!(format!( + "no token accounts for mint {mint}: burned nft?" + )))? + .address + .parse::()?) + } } diff --git a/ops/Cargo.toml b/ops/Cargo.toml index 63b91de67..1f37301b5 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -36,3 +36,5 @@ spl-account-compression = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +mpl-token-metadata = { workspace = true } +serde_json = { workspace = true } \ No newline at end of file diff --git a/ops/src/account/cmd.rs b/ops/src/account/cmd.rs index 659758d1c..9861cd8fe 100644 --- a/ops/src/account/cmd.rs +++ b/ops/src/account/cmd.rs @@ -1,4 +1,4 @@ -use super::{program, single}; +use super::{nft, program, single}; use anyhow::Result; use clap::{Args, Subcommand}; @@ -10,6 +10,9 @@ pub enum Commands { /// The 'single' command is used to backfill the index against a single account. #[clap(name = "single")] Single(single::Args), + + #[clap(name = "nft")] + Nft(nft::Args), } #[derive(Debug, Clone, Args)] @@ -26,6 +29,9 @@ pub async fn subcommand(subcommand: AccountCommand) -> Result<()> { Commands::Single(args) => { single::run(args).await?; } + Commands::Nft(args) => { + nft::run(args).await?; + } } Ok(()) diff --git a/ops/src/account/mod.rs b/ops/src/account/mod.rs index 59563b02e..e770cd362 100644 --- a/ops/src/account/mod.rs +++ b/ops/src/account/mod.rs @@ -1,5 +1,6 @@ mod account_info; mod cmd; +mod nft; mod program; mod single; diff --git a/ops/src/account/nft.rs b/ops/src/account/nft.rs new file mode 100644 index 000000000..f92e005d0 --- /dev/null +++ b/ops/src/account/nft.rs @@ -0,0 +1,95 @@ +use std::sync::Arc; + +use anyhow::Result; +use tokio::task::JoinHandle; + +use super::account_info; +use log::error; + +use clap::Parser; +use das_core::{ + connect_db, create_download_metadata_notifier, MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, + SolanaRpcArgs, +}; +use mpl_token_metadata::accounts::Metadata; +use program_transformers::ProgramTransformer; +use solana_sdk::pubkey::Pubkey; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, + + /// NFT Mint address + #[clap(value_parser = parse_pubkey)] + pub mint: Pubkey, +} + +fn parse_pubkey(s: &str) -> Result { + Pubkey::try_from(s).map_err(|_| "Failed to parse public key") +} + +pub async fn run(config: Args) -> Result<()> { + let rpc = Rpc::from_config(&config.solana); + let pool = connect_db(&config.database).await?; + let metadata_json_download_db_pool = pool.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = config + .metadata_json_download_worker + .start(metadata_json_download_db_pool)?; + + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender.clone()).await; + + let mint = config.mint; + + let metadata = Metadata::find_pda(&mint).0; + + let mut accounts_to_fetch = vec![mint, metadata]; + + let token_account = rpc.get_token_largest_account(mint).await; + + if let Ok(token_account) = token_account { + accounts_to_fetch.push(token_account); + } + + let program_transformer = Arc::new(ProgramTransformer::new(pool, download_metadata_notifier)); + let mut tasks = Vec::new(); + + for account in accounts_to_fetch { + let program_transformer = Arc::clone(&program_transformer); + let rpc = rpc.clone(); + + let task: JoinHandle> = tokio::spawn(async move { + let account_info = account_info::fetch(&rpc, account).await?; + if let Err(e) = program_transformer + .handle_account_update(&account_info) + .await + { + error!("Failed to handle account update: {:?}", e); + } + + Ok(()) + }); + + tasks.push(task); + } + + futures::future::try_join_all(tasks).await?; + + drop(metadata_json_download_sender); + + drop(program_transformer); + + metadata_json_download_worker.await?; + + Ok(()) +} From d7cc0587a109281abc1b6fdbf646a2350a2000a4 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 23 Oct 2024 19:54:08 +0400 Subject: [PATCH 58/72] Add lock timeout on v1_asset statements --- .../src/mpl_core_program/v1_asset.rs | 12 +++++++++++- program_transformers/src/token_metadata/v1_asset.rs | 13 ++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/program_transformers/src/mpl_core_program/v1_asset.rs b/program_transformers/src/mpl_core_program/v1_asset.rs index d651739c0..fd7d84f07 100644 --- a/program_transformers/src/mpl_core_program/v1_asset.rs +++ b/program_transformers/src/mpl_core_program/v1_asset.rs @@ -28,7 +28,7 @@ use { query::{JsonValue, QueryFilter, QueryTrait}, sea_query::query::OnConflict, sea_query::Expr, - ConnectionTrait, CursorTrait, DbBackend, TransactionTrait, + ConnectionTrait, CursorTrait, DbBackend, Statement, TransactionTrait, }, serde_json::{value::Value, Map}, solana_sdk::pubkey::Pubkey, @@ -108,6 +108,16 @@ pub async fn save_v1_asset( let txn = conn.begin().await?; + let set_lock_timeout = "SET LOCAL lock_timeout = '5s';"; + let set_local_app_name = + "SET LOCAL application_name = 'das::program_transformers::mpl_core_program::v1_asset';"; + let set_lock_timeout_stmt = + Statement::from_string(txn.get_database_backend(), set_lock_timeout.to_string()); + let set_local_app_name_stmt = + Statement::from_string(txn.get_database_backend(), set_local_app_name.to_string()); + txn.execute(set_lock_timeout_stmt).await?; + txn.execute(set_local_app_name_stmt).await?; + let model = asset_authority::ActiveModel { asset_id: ActiveValue::Set(id_vec.clone()), authority: ActiveValue::Set(update_authority.clone()), diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index 446654f10..cbb3d5c2c 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -28,7 +28,7 @@ use { entity::{ActiveValue, ColumnTrait, EntityTrait}, query::{JsonValue, Order, QueryFilter, QueryOrder, QueryTrait}, sea_query::query::OnConflict, - ConnectionTrait, DbBackend, DbErr, TransactionTrait, + ConnectionTrait, DbBackend, DbErr, Statement, TransactionTrait, }, solana_sdk::pubkey, sqlx::types::Decimal, @@ -235,6 +235,17 @@ pub async fn save_v1_asset( base_info_seq: ActiveValue::Set(Some(0)), }; let txn = conn.begin().await?; + + let set_lock_timeout = "SET LOCAL lock_timeout = '1ms';"; + let set_local_app_name = + "SET LOCAL application_name = 'das::program_transformers::token_metadata::v1_asset';"; + let set_lock_timeout_stmt = + Statement::from_string(txn.get_database_backend(), set_lock_timeout.to_string()); + let set_local_app_name_stmt = + Statement::from_string(txn.get_database_backend(), set_local_app_name.to_string()); + txn.execute(set_lock_timeout_stmt).await?; + txn.execute(set_local_app_name_stmt).await?; + let mut query = asset_data::Entity::insert(asset_data_model) .on_conflict( OnConflict::columns([asset_data::Column::Id]) From 17d7e0a5ca586547846158736d716413ea140fef Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 23 Oct 2024 19:56:26 +0400 Subject: [PATCH 59/72] Set the lock timeout to 5s --- program_transformers/src/token_metadata/v1_asset.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index cbb3d5c2c..2c3a38149 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -236,7 +236,7 @@ pub async fn save_v1_asset( }; let txn = conn.begin().await?; - let set_lock_timeout = "SET LOCAL lock_timeout = '1ms';"; + let set_lock_timeout = "SET LOCAL lock_timeout = '5s';"; let set_local_app_name = "SET LOCAL application_name = 'das::program_transformers::token_metadata::v1_asset';"; let set_lock_timeout_stmt = From 99b59c0b25381aef3fb713fe0c275c66a3e6f6c4 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Tue, 29 Oct 2024 10:01:15 +0100 Subject: [PATCH 60/72] Bubblegum verify (#150) * feat: move backfill to bubblegum crate and verfiy to it * feat: grpc ingest monitor --- Cargo.lock | 40 ++-- Cargo.toml | 9 +- {backfill => bubblegum}/Cargo.toml | 3 +- {backfill => bubblegum}/README.md | 0 .../src => bubblegum/src/backfill}/gap.rs | 3 +- bubblegum/src/backfill/mod.rs | 2 + .../src/backfill}/worker/gap.rs | 5 +- .../src/backfill}/worker/mod.rs | 0 .../backfill}/worker/program_transformer.rs | 4 +- .../src/backfill}/worker/transaction.rs | 8 +- .../src/backfill}/worker/tree.rs | 10 +- {backfill => bubblegum}/src/error.rs | 0 {backfill => bubblegum}/src/lib.rs | 52 +++-- {backfill => bubblegum}/src/tree.rs | 5 +- bubblegum/src/verify.rs | 177 ++++++++++++++++++ grpc-ingest/Cargo.toml | 1 + grpc-ingest/config-monitor.yml | 8 + grpc-ingest/src/config.rs | 46 ++++- grpc-ingest/src/main.rs | 11 ++ grpc-ingest/src/monitor.rs | 47 +++++ grpc-ingest/src/postgres.rs | 4 +- grpc-ingest/src/prom.rs | 53 ++++++ ops/Cargo.toml | 5 +- ops/src/bubblegum/backfiller.rs | 8 +- ops/src/bubblegum/cmd.rs | 7 +- ops/src/bubblegum/mod.rs | 1 + ops/src/bubblegum/replay.rs | 4 +- ops/src/bubblegum/verify.rs | 42 +++++ 28 files changed, 488 insertions(+), 67 deletions(-) rename {backfill => bubblegum}/Cargo.toml (95%) rename {backfill => bubblegum}/README.md (100%) rename {backfill/src => bubblegum/src/backfill}/gap.rs (98%) create mode 100644 bubblegum/src/backfill/mod.rs rename {backfill/src => bubblegum/src/backfill}/worker/gap.rs (94%) rename {backfill/src => bubblegum/src/backfill}/worker/mod.rs (100%) rename {backfill/src => bubblegum/src/backfill}/worker/program_transformer.rs (96%) rename {backfill/src => bubblegum/src/backfill}/worker/transaction.rs (97%) rename {backfill/src => bubblegum/src/backfill}/worker/tree.rs (95%) rename {backfill => bubblegum}/src/error.rs (100%) rename {backfill => bubblegum}/src/lib.rs (78%) rename {backfill => bubblegum}/src/tree.rs (98%) create mode 100644 bubblegum/src/verify.rs create mode 100644 grpc-ingest/config-monitor.yml create mode 100644 grpc-ingest/src/monitor.rs create mode 100644 ops/src/bubblegum/verify.rs diff --git a/Cargo.lock b/Cargo.lock index 856a5c9a2..d3563d3b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,7 @@ dependencies = [ "anchor-derive-accounts", "anchor-derive-serde", "anchor-derive-space", + "anchor-syn", "arrayref", "base64 0.13.1", "bincode", @@ -1097,7 +1098,7 @@ dependencies = [ "solana-transaction-status", "solana-zk-token-sdk", "spl-account-compression", - "spl-concurrent-merkle-tree", + "spl-concurrent-merkle-tree 0.2.0", "spl-noop", "spl-pod", "spl-token", @@ -1323,9 +1324,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] @@ -1816,7 +1817,7 @@ dependencies = [ ] [[package]] -name = "das-backfill" +name = "das-bubblegum" version = "0.7.2" dependencies = [ "anchor-client", @@ -1835,6 +1836,7 @@ dependencies = [ "program_transformers", "sea-orm", "serde_json", + "sha3 0.10.8", "solana-client", "solana-program", "solana-sdk", @@ -1890,6 +1892,7 @@ dependencies = [ "cargo-lock", "chrono", "clap 4.4.8", + "das-bubblegum", "das-core", "digital_asset_types", "futures", @@ -1936,7 +1939,7 @@ dependencies = [ "cadence", "cadence-macros", "clap 4.4.8", - "das-backfill", + "das-bubblegum", "das-core", "digital_asset_types", "env_logger 0.10.0", @@ -1958,6 +1961,7 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tracing", ] [[package]] @@ -2138,7 +2142,7 @@ dependencies = [ "serde", "serde_json", "solana-sdk", - "spl-concurrent-merkle-tree", + "spl-concurrent-merkle-tree 0.2.0", "thiserror", "tokio", "url", @@ -3766,9 +3770,9 @@ dependencies = [ [[package]] name = "mpl-bubblegum" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3cbca5deb859e66a1a21ada94f2eaab3eb5caa4584c0c8ade0efac29a5414b8" +checksum = "a9eff5ae5cafd1acdf7e7c93359da1eec91dcaede318470d9f68b78e8b7469f4" dependencies = [ "borsh 0.10.3", "kaigan", @@ -4066,7 +4070,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 2.0.66", @@ -6965,13 +6969,14 @@ dependencies = [ [[package]] name = "spl-account-compression" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c43bd4455d9fb29b9e4f83c087ccffa2f6f41fecfc0549932ae391d00f3378" +checksum = "8ce8314ec6ae26084ec7c6c0802c3dc173ee86aee5f5d5026a3f82c52cfe1c07" dependencies = [ "anchor-lang", "bytemuck", - "spl-concurrent-merkle-tree", + "solana-program", + "spl-concurrent-merkle-tree 0.4.0", "spl-noop", ] @@ -7002,6 +7007,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "spl-concurrent-merkle-tree" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d1bbb97252d8a1b90d3d56425038928382a306b71dbba4c836973c94b33f96" +dependencies = [ + "bytemuck", + "solana-program", + "thiserror", +] + [[package]] name = "spl-discriminator" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 8a77a8473..b0b42139d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ - "backfill", + "bubblegum", "blockbuster", "core", "das_api", @@ -48,7 +48,7 @@ cargo-lock = "9.0.0" chrono = "0.4.19" clap = "4.2.2" das-core = { path = "core" } -das-backfill = { path = "backfill" } +das-bubblegum = { path = "bubblegum" } das_api = { path = "das_api" } derive_more = { version = "0.99.17" } digital_asset_types = { path = "digital_asset_types" } @@ -78,7 +78,7 @@ lru = "0.12.3" metrics = "0.20.1" migration = { path = "migration" } mime_guess = "2.0.4" -mpl-bubblegum = "1.2.0" +mpl-bubblegum = "1.4.0" mpl-core = { version = "0.8.0-beta.1", features = ["serde"] } mpl-token-metadata = "4.1.1" nft_ingester = { path = "nft_ingester" } @@ -116,7 +116,7 @@ solana-program = "~1.18" solana-sdk = "~1.18" solana-transaction-status = "~1.18" solana-zk-token-sdk = "1.17.16" -spl-account-compression = "0.3.0" +spl-account-compression = "0.4.0" spl-associated-token-account = ">= 1.1.3, < 3.0" spl-concurrent-merkle-tree = "0.2.0" spl-noop = "0.2.0" @@ -125,6 +125,7 @@ spl-token = ">= 3.5.0, < 5.0" spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] } spl-token-group-interface = "0.1.0" spl-token-metadata-interface = "0.2.0" +sha3 = "0.10.8" sqlx = "0.6.2" stretto = "0.8.4" thiserror = "1.0.31" diff --git a/backfill/Cargo.toml b/bubblegum/Cargo.toml similarity index 95% rename from backfill/Cargo.toml rename to bubblegum/Cargo.toml index ce082f81c..0ecd4704a 100644 --- a/backfill/Cargo.toml +++ b/bubblegum/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "das-backfill" +name = "das-bubblegum" version = { workspace = true } edition = { workspace = true } repository = { workspace = true } @@ -28,6 +28,7 @@ num-traits = { workspace = true } sea-orm = { workspace = true } serde_json = { workspace = true } solana-sdk = { workspace = true } +sha3 = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true, features = ["no-entrypoint"] } spl-token = { workspace = true, features = ["no-entrypoint"] } diff --git a/backfill/README.md b/bubblegum/README.md similarity index 100% rename from backfill/README.md rename to bubblegum/README.md diff --git a/backfill/src/gap.rs b/bubblegum/src/backfill/gap.rs similarity index 98% rename from backfill/src/gap.rs rename to bubblegum/src/backfill/gap.rs index feb523b98..e167e888d 100644 --- a/backfill/src/gap.rs +++ b/bubblegum/src/backfill/gap.rs @@ -1,5 +1,4 @@ -use super::ErrorKind; -use crate::Rpc; +use crate::{error::ErrorKind, Rpc}; use anyhow::Result; use clap::Args; use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; diff --git a/bubblegum/src/backfill/mod.rs b/bubblegum/src/backfill/mod.rs new file mode 100644 index 000000000..5a4a874d6 --- /dev/null +++ b/bubblegum/src/backfill/mod.rs @@ -0,0 +1,2 @@ +pub mod gap; +pub mod worker; diff --git a/backfill/src/worker/gap.rs b/bubblegum/src/backfill/worker/gap.rs similarity index 94% rename from backfill/src/worker/gap.rs rename to bubblegum/src/backfill/worker/gap.rs index 07b88f4f5..68523fdbe 100644 --- a/backfill/src/worker/gap.rs +++ b/bubblegum/src/backfill/worker/gap.rs @@ -9,8 +9,7 @@ use tokio::{ task::JoinHandle, }; -use crate::gap::TreeGapFill; -use crate::BubblegumBackfillContext; +use crate::{backfill::gap::TreeGapFill, BubblegumContext}; #[derive(Parser, Debug, Clone)] pub struct GapWorkerArgs { @@ -26,7 +25,7 @@ pub struct GapWorkerArgs { impl GapWorkerArgs { pub fn start( &self, - context: BubblegumBackfillContext, + context: BubblegumContext, forward: Sender, ) -> Result<(JoinHandle<()>, Sender)> { let (gap_sender, mut gap_receiver) = channel::(self.gap_channel_size); diff --git a/backfill/src/worker/mod.rs b/bubblegum/src/backfill/worker/mod.rs similarity index 100% rename from backfill/src/worker/mod.rs rename to bubblegum/src/backfill/worker/mod.rs diff --git a/backfill/src/worker/program_transformer.rs b/bubblegum/src/backfill/worker/program_transformer.rs similarity index 96% rename from backfill/src/worker/program_transformer.rs rename to bubblegum/src/backfill/worker/program_transformer.rs index 6e7c20646..75f75c0a7 100644 --- a/backfill/src/worker/program_transformer.rs +++ b/bubblegum/src/backfill/worker/program_transformer.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender, UnboundedSender}; use tokio::task::JoinHandle; -use crate::BubblegumBackfillContext; +use crate::BubblegumContext; #[derive(Parser, Debug, Clone)] pub struct ProgramTransformerWorkerArgs { @@ -22,7 +22,7 @@ pub struct ProgramTransformerWorkerArgs { impl ProgramTransformerWorkerArgs { pub fn start( &self, - context: BubblegumBackfillContext, + context: BubblegumContext, forwarder: UnboundedSender, ) -> Result<(JoinHandle<()>, Sender)> { let (sender, mut receiver) = diff --git a/backfill/src/worker/transaction.rs b/bubblegum/src/backfill/worker/transaction.rs similarity index 97% rename from backfill/src/worker/transaction.rs rename to bubblegum/src/backfill/worker/transaction.rs index 6a25f6f1b..910b79a92 100644 --- a/backfill/src/worker/transaction.rs +++ b/bubblegum/src/backfill/worker/transaction.rs @@ -1,4 +1,4 @@ -use crate::error::ErrorKind; +use crate::{error::ErrorKind, BubblegumContext}; use anyhow::Result; use clap::Parser; use das_core::Rpc; @@ -143,11 +143,13 @@ pub struct SignatureWorkerArgs { pub signature_worker_count: usize, } +type TransactionSender = Sender; + impl SignatureWorkerArgs { pub fn start( &self, - context: crate::BubblegumBackfillContext, - forwarder: Sender, + context: BubblegumContext, + forwarder: TransactionSender, ) -> Result<(JoinHandle<()>, Sender)> { let (sig_sender, mut sig_receiver) = channel::(self.signature_channel_size); let worker_count = self.signature_worker_count; diff --git a/backfill/src/worker/tree.rs b/bubblegum/src/backfill/worker/tree.rs similarity index 95% rename from backfill/src/worker/tree.rs rename to bubblegum/src/backfill/worker/tree.rs index e6db2b8d9..e02c21305 100644 --- a/backfill/src/worker/tree.rs +++ b/bubblegum/src/backfill/worker/tree.rs @@ -1,7 +1,7 @@ use crate::{ - gap::{TreeGapFill, TreeGapModel}, + backfill::gap::{TreeGapFill, TreeGapModel}, tree::TreeResponse, - BubblegumBackfillContext, + BubblegumContext, }; use anyhow::Result; use clap::Parser; @@ -32,11 +32,7 @@ pub struct TreeWorkerArgs { pub force: bool, } impl TreeWorkerArgs { - pub fn start( - &self, - context: BubblegumBackfillContext, - tree: TreeResponse, - ) -> JoinHandle> { + pub fn start(&self, context: BubblegumContext, tree: TreeResponse) -> JoinHandle> { let db_pool = context.database_pool.clone(); let metadata_json_download_db_pool = context.database_pool.clone(); diff --git a/backfill/src/error.rs b/bubblegum/src/error.rs similarity index 100% rename from backfill/src/error.rs rename to bubblegum/src/error.rs diff --git a/backfill/src/lib.rs b/bubblegum/src/lib.rs similarity index 78% rename from backfill/src/lib.rs rename to bubblegum/src/lib.rs index dac3aba44..f1dc031c6 100644 --- a/backfill/src/lib.rs +++ b/bubblegum/src/lib.rs @@ -1,12 +1,14 @@ +mod backfill; mod error; -mod gap; mod tree; -mod worker; use das_core::{MetadataJsonDownloadWorkerArgs, Rpc}; pub use error::ErrorKind; +mod verify; +pub use verify::ProofReport; use anyhow::Result; +use backfill::worker::{ProgramTransformerWorkerArgs, SignatureWorkerArgs, TreeWorkerArgs}; use clap::Parser; use digital_asset_types::dao::cl_audits_v2; use futures::{stream::FuturesUnordered, StreamExt}; @@ -19,16 +21,14 @@ use solana_sdk::signature::Signature; use std::str::FromStr; use tracing::error; use tree::TreeResponse; -use worker::ProgramTransformerWorkerArgs; -use worker::{SignatureWorkerArgs, TreeWorkerArgs}; #[derive(Clone)] -pub struct BubblegumBackfillContext { +pub struct BubblegumContext { pub database_pool: sqlx::PgPool, pub solana_rpc: Rpc, } -impl BubblegumBackfillContext { +impl BubblegumContext { pub const fn new(database_pool: sqlx::PgPool, solana_rpc: Rpc) -> Self { Self { database_pool, @@ -38,7 +38,7 @@ impl BubblegumBackfillContext { } #[derive(Debug, Parser, Clone)] -pub struct BubblegumBackfillArgs { +pub struct BackfillArgs { /// Number of tree crawler workers #[arg(long, env, default_value = "20")] pub tree_crawler_count: usize, @@ -51,10 +51,7 @@ pub struct BubblegumBackfillArgs { pub tree_worker: TreeWorkerArgs, } -pub async fn start_bubblegum_backfill( - context: BubblegumBackfillContext, - args: BubblegumBackfillArgs, -) -> Result<()> { +pub async fn start_backfill(context: BubblegumContext, args: BackfillArgs) -> Result<()> { let trees = if let Some(ref only_trees) = args.only_trees { TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? } else { @@ -99,7 +96,7 @@ pub struct BubblegumReplayArgs { } pub async fn start_bubblegum_replay( - context: BubblegumBackfillContext, + context: BubblegumContext, args: BubblegumReplayArgs, ) -> Result<()> { let pubkey = Pubkey::from_str(&args.tree) @@ -154,3 +151,34 @@ pub async fn start_bubblegum_replay( Ok(()) } + +#[derive(Debug, Parser, Clone)] +pub struct VerifyArgs { + /// The list of trees to verify. If not specified, all trees will be crawled. + #[arg(long, env, use_value_delimiter = true)] + pub only_trees: Option>, + + #[arg(long, env, default_value = "20")] + pub max_concurrency: usize, +} + +pub async fn verify_bubblegum( + context: BubblegumContext, + args: VerifyArgs, +) -> Result> { + let trees = if let Some(ref only_trees) = args.only_trees { + TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? + } else { + TreeResponse::all(&context.solana_rpc).await? + }; + + let mut reports = Vec::new(); + + for tree in trees { + let report = verify::check(context.clone(), tree, args.max_concurrency).await?; + + reports.push(report); + } + + Ok(reports) +} diff --git a/backfill/src/tree.rs b/bubblegum/src/tree.rs similarity index 98% rename from backfill/src/tree.rs rename to bubblegum/src/tree.rs index efb5c1cbf..405c90037 100644 --- a/backfill/src/tree.rs +++ b/bubblegum/src/tree.rs @@ -1,4 +1,4 @@ -use super::ErrorKind; +use super::error::ErrorKind; use anyhow::Result; use borsh::BorshDeserialize; use das_core::Rpc; @@ -10,7 +10,7 @@ use spl_account_compression::state::{ }; use std::str::FromStr; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct TreeHeaderResponse { pub max_depth: u32, pub max_buffer_size: u32, @@ -33,7 +33,6 @@ impl TryFrom for TreeHeaderResponse { } } -#[derive(Debug, Clone)] pub struct TreeResponse { pub pubkey: Pubkey, pub tree_header: TreeHeaderResponse, diff --git a/bubblegum/src/verify.rs b/bubblegum/src/verify.rs new file mode 100644 index 000000000..fae6051d6 --- /dev/null +++ b/bubblegum/src/verify.rs @@ -0,0 +1,177 @@ +use super::BubblegumContext; +use crate::error::ErrorKind; +use crate::tree::TreeResponse; +use anyhow::{anyhow, Result}; +use digital_asset_types::dapi::get_proof_for_asset; +use digital_asset_types::rpc::AssetProof; +use futures::stream::{FuturesUnordered, StreamExt}; +use mpl_bubblegum::accounts::TreeConfig; +use sea_orm::SqlxPostgresConnector; +use sha3::{Digest, Keccak256}; +use solana_sdk::pubkey::Pubkey; +use spl_account_compression::concurrent_tree_wrapper::ProveLeafArgs; +use std::fmt; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing::error; + +trait TryFromAssetProof { + fn try_from_asset_proof(proof: AssetProof) -> Result + where + Self: Sized; +} + +impl TryFromAssetProof for ProveLeafArgs { + fn try_from_asset_proof(proof: AssetProof) -> Result { + Ok(ProveLeafArgs { + current_root: bs58::decode(&proof.root) + .into_vec() + .map_err(|e| anyhow!(e))? + .try_into() + .map_err(|_| anyhow!("Invalid root length"))?, + leaf: bs58::decode(&proof.leaf) + .into_vec() + .map_err(|e| anyhow!(e))? + .try_into() + .map_err(|_| anyhow!("Invalid leaf length"))?, + proof_vec: proof + .proof + .iter() + .map(|p| { + bs58::decode(p) + .into_vec() + .map_err(|e| anyhow!(e)) + .and_then(|v| v.try_into().map_err(|_| anyhow!("Invalid proof length"))) + }) + .collect::>>()?, + index: proof.node_index as u32, + }) + } +} + +fn hash(left: &[u8], right: &[u8]) -> [u8; 32] { + let mut hasher = Keccak256::new(); + hasher.update(left); + hasher.update(right); + let result = hasher.finalize(); + let mut hash = [0u8; 32]; + hash.copy_from_slice(&result); + hash +} + +fn verify_merkle_proof(proof: &ProveLeafArgs) -> bool { + let mut node = proof.leaf; + for (i, sibling) in proof.proof_vec.iter().enumerate() { + if (proof.index >> i) & 1 == 0 { + node = hash(&node, sibling); + } else { + node = hash(sibling, &node); + } + } + node == proof.current_root +} + +fn leaf_proof_result(proof: AssetProof) -> Result { + match ProveLeafArgs::try_from_asset_proof(proof) { + Ok(proof) if verify_merkle_proof(&proof) => Ok(ProofResult::Correct), + Ok(_) => Ok(ProofResult::Incorrect), + Err(_) => Ok(ProofResult::Corrupt), + } +} + +#[derive(Debug, Default)] +pub struct ProofReport { + pub tree_pubkey: Pubkey, + pub total_leaves: usize, + pub incorrect_proofs: usize, + pub not_found_proofs: usize, + pub correct_proofs: usize, + pub corrupt_proofs: usize, +} + +enum ProofResult { + Correct, + Incorrect, + NotFound, + Corrupt, +} + +impl fmt::Display for ProofResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProofResult::Correct => write!(f, "Correct proof found"), + ProofResult::Incorrect => write!(f, "Incorrect proof found"), + ProofResult::NotFound => write!(f, "Proof not found"), + ProofResult::Corrupt => write!(f, "Corrupt proof found"), + } + } +} + +pub async fn check( + context: BubblegumContext, + tree: TreeResponse, + max_concurrency: usize, +) -> Result { + let (tree_config_pubkey, _) = TreeConfig::find_pda(&tree.pubkey); + + let pool = context.database_pool.clone(); + + let account = context.solana_rpc.get_account(&tree_config_pubkey).await?; + let account = account + .value + .ok_or_else(|| ErrorKind::Generic("Account not found".to_string()))?; + + let tree_config = TreeConfig::from_bytes(account.data.as_slice())?; + + let report = Arc::new(Mutex::new(ProofReport { + tree_pubkey: tree.pubkey, + total_leaves: tree_config.num_minted as usize, + ..ProofReport::default() + })); + + let mut tasks = FuturesUnordered::new(); + + for i in 0..tree_config.num_minted { + if tasks.len() >= max_concurrency { + tasks.next().await; + } + + let db = SqlxPostgresConnector::from_sqlx_postgres_pool(pool.clone()); + let tree_pubkey = tree.pubkey; + let report = Arc::clone(&report); + + tasks.push(tokio::spawn(async move { + let (asset, _) = Pubkey::find_program_address( + &[b"asset", &tree_pubkey.to_bytes(), &i.to_le_bytes()], + &mpl_bubblegum::ID, + ); + let proof_lookup: Result = + get_proof_for_asset(&db, asset.to_bytes().to_vec()) + .await + .map_or_else(|_| Ok(ProofResult::NotFound), leaf_proof_result); + + if let Ok(proof_result) = proof_lookup { + let mut report = report.lock().await; + match proof_result { + ProofResult::Correct => report.correct_proofs += 1, + ProofResult::Incorrect => report.incorrect_proofs += 1, + ProofResult::NotFound => report.not_found_proofs += 1, + ProofResult::Corrupt => report.corrupt_proofs += 1, + } + if let ProofResult::Incorrect | ProofResult::NotFound | ProofResult::Corrupt = + proof_result + { + error!(tree = %tree_pubkey, leaf_index = i, asset = %asset, "{}", proof_result); + } + } + })); + } + + while tasks.next().await.is_some() {} + + let final_report = Arc::try_unwrap(report) + .expect("Failed to unwrap Arc") + .into_inner(); + + Ok(final_report) +} diff --git a/grpc-ingest/Cargo.toml b/grpc-ingest/Cargo.toml index b4237fdc2..4036be057 100644 --- a/grpc-ingest/Cargo.toml +++ b/grpc-ingest/Cargo.toml @@ -9,6 +9,7 @@ publish = { workspace = true } anyhow = { workspace = true } async-stream = { workspace = true } atty = { workspace = true } +das-bubblegum = { workspace = true } sqlx = { workspace = true, features = [ "macros", "runtime-tokio-rustls", diff --git a/grpc-ingest/config-monitor.yml b/grpc-ingest/config-monitor.yml new file mode 100644 index 000000000..8b3278864 --- /dev/null +++ b/grpc-ingest/config-monitor.yml @@ -0,0 +1,8 @@ +prometheus: 127.0.0.1:8876 +rpc: http://127.0.0.1:8899 +postgres: + url: postgres://solana:solana@localhost/solana + min_connections: 10 + max_connections: 50 +bubblegum: + only_trees: null diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 64ef42953..cab28ad6c 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -214,7 +214,7 @@ where #[derive(Debug, Clone, Deserialize)] pub struct ConfigIngester { pub redis: String, - pub postgres: ConfigIngesterPostgres, + pub postgres: ConfigPostgres, pub download_metadata: ConfigIngesterDownloadMetadata, pub snapshots: ConfigIngestStream, pub accounts: ConfigIngestStream, @@ -231,31 +231,31 @@ pub enum ConfigIngesterRedisStreamType { } #[derive(Debug, Clone, Deserialize)] -pub struct ConfigIngesterPostgres { +pub struct ConfigPostgres { pub url: String, #[serde( - default = "ConfigIngesterPostgres::default_min_connections", + default = "ConfigPostgres::default_min_connections", deserialize_with = "deserialize_usize_str" )] pub min_connections: usize, #[serde( - default = "ConfigIngesterPostgres::default_max_connections", + default = "ConfigPostgres::default_max_connections", deserialize_with = "deserialize_usize_str" )] pub max_connections: usize, #[serde( - default = "ConfigIngesterPostgres::default_idle_timeout", + default = "ConfigPostgres::default_idle_timeout", deserialize_with = "deserialize_duration_str" )] pub idle_timeout: Duration, #[serde( - default = "ConfigIngesterPostgres::default_max_lifetime", + default = "ConfigPostgres::default_max_lifetime", deserialize_with = "deserialize_duration_str" )] pub max_lifetime: Duration, } -impl ConfigIngesterPostgres { +impl ConfigPostgres { pub const fn default_min_connections() -> usize { 10 } @@ -335,3 +335,35 @@ impl ConfigIngesterDownloadMetadata { Duration::from_millis(3_000) } } + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigMonitor { + pub postgres: ConfigPostgres, + pub rpc: String, + pub bubblegum: ConfigBubblegumVerify, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigBubblegumVerify { + #[serde( + default = "ConfigBubblegumVerify::default_report_interval", + deserialize_with = "deserialize_duration_str" + )] + pub report_interval: Duration, + #[serde(default)] + pub only_trees: Option>, + #[serde( + default = "ConfigBubblegumVerify::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, +} + +impl ConfigBubblegumVerify { + pub const fn default_report_interval() -> Duration { + Duration::from_millis(5 * 60 * 1000) + } + pub const fn default_max_concurrency() -> usize { + 20 + } +} diff --git a/grpc-ingest/src/main.rs b/grpc-ingest/src/main.rs index 907e1e00c..99ad667be 100644 --- a/grpc-ingest/src/main.rs +++ b/grpc-ingest/src/main.rs @@ -6,12 +6,14 @@ use { }, anyhow::Context, clap::{Parser, Subcommand}, + config::ConfigMonitor, std::net::SocketAddr, }; mod config; mod grpc; mod ingester; +mod monitor; mod postgres; mod prom; mod redis; @@ -42,6 +44,8 @@ enum ArgsAction { /// Run ingester process (process events from Redis) #[command(name = "ingester")] Ingester, + #[command(name = "monitor")] + Monitor, } #[tokio::main] @@ -72,5 +76,12 @@ async fn main() -> anyhow::Result<()> { .with_context(|| format!("failed to parse config from: {}", args.config))?; ingester::run(config).await } + ArgsAction::Monitor => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + + monitor::run(config).await + } } } diff --git a/grpc-ingest/src/monitor.rs b/grpc-ingest/src/monitor.rs new file mode 100644 index 000000000..ac0c687ec --- /dev/null +++ b/grpc-ingest/src/monitor.rs @@ -0,0 +1,47 @@ +use crate::postgres::create_pool; +use crate::util::create_shutdown; +use crate::{config::ConfigMonitor, prom::update_tree_proof_report}; +use das_bubblegum::{verify_bubblegum, BubblegumContext, VerifyArgs}; +use das_core::{Rpc, SolanaRpcArgs}; +use futures::stream::StreamExt; +use tracing::error; + +pub async fn run(config: ConfigMonitor) -> anyhow::Result<()> { + let mut shutdown = create_shutdown()?; + let database_pool = create_pool(config.postgres).await?; + let rpc = Rpc::from_config(&SolanaRpcArgs { + solana_rpc_url: config.rpc, + }); + + let bubblegum_verify = tokio::spawn(async move { + loop { + let bubblegum_context = BubblegumContext::new(database_pool.clone(), rpc.clone()); + let verify_args = VerifyArgs { + only_trees: config.bubblegum.only_trees.clone(), + max_concurrency: config.bubblegum.max_concurrency, + }; + + match verify_bubblegum(bubblegum_context, verify_args).await { + Ok(reports) => { + for report in reports { + update_tree_proof_report(&report); + } + } + Err(e) => { + error!( + message = "Error verifying bubblegum", + error = ?e + ); + } + } + + tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; + } + }); + + if let Some(_signal) = shutdown.next().await {} + + bubblegum_verify.abort(); + + Ok(()) +} diff --git a/grpc-ingest/src/postgres.rs b/grpc-ingest/src/postgres.rs index 1077bc49b..d38c5250c 100644 --- a/grpc-ingest/src/postgres.rs +++ b/grpc-ingest/src/postgres.rs @@ -1,6 +1,6 @@ use { crate::{ - config::ConfigIngesterPostgres, + config::ConfigPostgres, prom::{pgpool_connections_set, PgpoolConnectionsKind}, }, sqlx::{ @@ -9,7 +9,7 @@ use { }, }; -pub async fn create_pool(config: ConfigIngesterPostgres) -> anyhow::Result { +pub async fn create_pool(config: ConfigPostgres) -> anyhow::Result { let options: PgConnectOptions = config.url.parse()?; PgPoolOptions::new() .min_connections(config.min_connections.try_into()?) diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index e70e14671..a82022796 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -1,5 +1,6 @@ use { crate::{redis::RedisStreamMessageError, version::VERSION as VERSION_INFO}, + das_bubblegum::ProofReport, das_core::MetadataJsonTaskError, hyper::{ server::conn::AddrStream, @@ -72,6 +73,31 @@ lazy_static::lazy_static! { Opts::new("grpc_tasks", "Number of tasks spawned for writing grpc messages to redis "), &[] ).unwrap(); + + static ref BUBBLEGUM_TREE_TOTAL_LEAVES: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_total_leaves", "Total number of leaves in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_INCORRECT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_incorrect_proofs", "Number of incorrect proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_NOT_FOUND_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_not_found_proofs", "Number of not found proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_CORRECT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_correct_proofs", "Number of correct proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_CORRUPT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_corrupt_proofs", "Number of corrupt proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); } pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { @@ -96,6 +122,11 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(INGEST_TASKS); register!(ACK_TASKS); register!(GRPC_TASKS); + register!(BUBBLEGUM_TREE_TOTAL_LEAVES); + register!(BUBBLEGUM_TREE_INCORRECT_PROOFS); + register!(BUBBLEGUM_TREE_NOT_FOUND_PROOFS); + register!(BUBBLEGUM_TREE_CORRECT_PROOFS); + register!(BUBBLEGUM_TREE_CORRUPT_PROOFS); VERSION_INFO_METRIC .with_label_values(&[ @@ -319,3 +350,25 @@ pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKin .with_label_values(&[kind.to_str()]) .inc() } + +pub fn update_tree_proof_report(report: &ProofReport) { + BUBBLEGUM_TREE_TOTAL_LEAVES + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.total_leaves as i64); + + BUBBLEGUM_TREE_INCORRECT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.incorrect_proofs as i64); + + BUBBLEGUM_TREE_NOT_FOUND_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.not_found_proofs as i64); + + BUBBLEGUM_TREE_CORRECT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.correct_proofs as i64); + + BUBBLEGUM_TREE_CORRUPT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.corrupt_proofs as i64); +} diff --git a/ops/Cargo.toml b/ops/Cargo.toml index 1f37301b5..14f741870 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -16,7 +16,7 @@ bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "env"] } -das-backfill = { workspace = true } +das-bubblegum = { workspace = true } das-core = { workspace = true } digital_asset_types = { workspace = true } env_logger = { workspace = true } @@ -36,5 +36,6 @@ spl-account-compression = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } mpl-token-metadata = { workspace = true } -serde_json = { workspace = true } \ No newline at end of file +serde_json = { workspace = true } diff --git a/ops/src/bubblegum/backfiller.rs b/ops/src/bubblegum/backfiller.rs index a53996e64..c13315aaa 100644 --- a/ops/src/bubblegum/backfiller.rs +++ b/ops/src/bubblegum/backfiller.rs @@ -1,13 +1,13 @@ use anyhow::Result; use clap::Parser; -use das_backfill::{start_bubblegum_backfill, BubblegumBackfillArgs, BubblegumBackfillContext}; +use das_bubblegum::{start_backfill, BackfillArgs, BubblegumContext}; use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; #[derive(Debug, Parser, Clone)] pub struct Args { /// Backfill Bubblegum Args #[clap(flatten)] - pub backfill_bubblegum: BubblegumBackfillArgs, + pub backfill_bubblegum: BackfillArgs, /// Database configuration #[clap(flatten)] @@ -47,7 +47,7 @@ pub async fn run(config: Args) -> Result<()> { let database_pool = connect_db(&config.database).await?; let solana_rpc = Rpc::from_config(&config.solana); - let context = BubblegumBackfillContext::new(database_pool, solana_rpc); + let context = BubblegumContext::new(database_pool, solana_rpc); - start_bubblegum_backfill(context, config.backfill_bubblegum).await + start_backfill(context, config.backfill_bubblegum).await } diff --git a/ops/src/bubblegum/cmd.rs b/ops/src/bubblegum/cmd.rs index 517c80835..dd19c3246 100644 --- a/ops/src/bubblegum/cmd.rs +++ b/ops/src/bubblegum/cmd.rs @@ -1,4 +1,4 @@ -use super::{backfiller, replay}; +use super::{backfiller, replay, verify}; use anyhow::Result; use clap::{Args, Subcommand}; @@ -10,6 +10,8 @@ pub enum Commands { Backfill(backfiller::Args), #[clap(name = "replay")] Replay(replay::Args), + /// The 'verify' command is used to verify the integrity of the bubblegum index. + Verify(verify::Args), } #[derive(Debug, Clone, Args)] @@ -26,6 +28,9 @@ pub async fn subcommand(subcommand: BubblegumCommand) -> Result<()> { Commands::Replay(args) => { replay::run(args).await?; } + Commands::Verify(args) => { + verify::run(args).await?; + } } Ok(()) diff --git a/ops/src/bubblegum/mod.rs b/ops/src/bubblegum/mod.rs index 6fd6d0efb..0f683a332 100644 --- a/ops/src/bubblegum/mod.rs +++ b/ops/src/bubblegum/mod.rs @@ -1,5 +1,6 @@ mod backfiller; mod cmd; mod replay; +mod verify; pub use cmd::*; diff --git a/ops/src/bubblegum/replay.rs b/ops/src/bubblegum/replay.rs index 23647cbbe..c777b913c 100644 --- a/ops/src/bubblegum/replay.rs +++ b/ops/src/bubblegum/replay.rs @@ -1,6 +1,6 @@ use anyhow::Result; use clap::Parser; -use das_backfill::{start_bubblegum_replay, BubblegumBackfillContext, BubblegumReplayArgs}; +use das_bubblegum::{start_bubblegum_replay, BubblegumContext, BubblegumReplayArgs}; use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; #[derive(Debug, Parser, Clone)] @@ -21,7 +21,7 @@ pub async fn run(config: Args) -> Result<()> { let database_pool = connect_db(&config.database).await?; let solana_rpc = Rpc::from_config(&config.solana); - let context = BubblegumBackfillContext::new(database_pool, solana_rpc); + let context = BubblegumContext::new(database_pool, solana_rpc); start_bubblegum_replay(context, config.replay_bubblegum).await } diff --git a/ops/src/bubblegum/verify.rs b/ops/src/bubblegum/verify.rs new file mode 100644 index 000000000..947d2b3ad --- /dev/null +++ b/ops/src/bubblegum/verify.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use clap::Parser; +use das_bubblegum::{verify_bubblegum, BubblegumContext, VerifyArgs}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; +use tracing::info; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Verify Bubblegum Args + #[clap(flatten)] + pub verify_bubblegum: VerifyArgs, + + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, +} + +pub async fn run(config: Args) -> Result<()> { + let database_pool = connect_db(&config.database).await?; + + let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumContext::new(database_pool, solana_rpc); + + let reports = verify_bubblegum(context, config.verify_bubblegum).await?; + + for report in reports { + info!( + "Tree: {}, Total Leaves: {}, Incorrect Proofs: {}, Not Found Proofs: {}, Correct Proofs: {}", + report.tree_pubkey, + report.total_leaves, + report.incorrect_proofs, + report.not_found_proofs, + report.correct_proofs + ); + } + + Ok(()) +} From d3bfe1b6ac48889818ae712c98344b145a4f100b Mon Sep 17 00:00:00 2001 From: Nagaprasad V R <81755170+Nagaprasadvr@users.noreply.github.com> Date: Fri, 8 Nov 2024 13:21:45 +0530 Subject: [PATCH 61/72] Separate global stream to multiple Program separated streams and handle each stream (#168) * feat : separate global stream to multiple streams depending on programs * WIP : add metrics for each program subscription * refactor: add structs with builder for SubscriptionTask and add better shutdown handling * refactor : separate subscription builder to work on each subscription * refactor: change pipeline to handle single stream instead of multiple streams * Update: readme for grpc-ingest, index-readme, ops (#172) * Update: readme for grpc-ingest, index-readme, ops * Update : README * fix: prometheus ports * chore: resolve comments * chore: add comments to config files in grpc-ingest (#173) * change tag name of subscriptions * Remove topograph from grpc2redis --------- Co-authored-by: Kyle Espinola --- Cargo.lock | 51 ---- Cargo.toml | 7 +- README.md | 11 + docker-compose.yaml | 19 +- grpc-ingest/Cargo.toml | 1 - grpc-ingest/README.md | 49 ++-- grpc-ingest/config-grpc2redis.yml | 89 ++++-- grpc-ingest/config-ingester.yml | 28 +- grpc-ingest/config-monitor.yml | 14 +- grpc-ingest/src/config.rs | 95 ++++--- grpc-ingest/src/grpc.rs | 440 ++++++++++++++++++------------ grpc-ingest/src/ingester.rs | 6 +- grpc-ingest/src/prom.rs | 28 +- grpc-ingest/src/redis.rs | 22 +- ops/README.md | 41 +++ ops/src/account/cmd.rs | 2 +- 16 files changed, 558 insertions(+), 345 deletions(-) create mode 100644 ops/README.md diff --git a/Cargo.lock b/Cargo.lock index d3563d3b0..b88325803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1676,19 +1676,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1918,7 +1905,6 @@ dependencies = [ "sqlx", "thiserror", "tokio", - "topograph", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2179,27 +2165,6 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "dispose" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a73c22432c5657d246abd8b3490c0678ed59549c474d96402367be7bd18700d1" -dependencies = [ - "dispose-derive", -] - -[[package]] -name = "dispose-derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62e5546deabd472871fc4de055f8d75c0a97c492f1889cc19623e98d20ba294" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "dlopen2" version = "0.5.0" @@ -7875,22 +7840,6 @@ dependencies = [ "tonic", ] -[[package]] -name = "topograph" -version = "0.4.0" -source = "git+https://github.com/ray-kast/topograph?branch=v0.4-wip#c91c01c25f605c4bc4461e9b9cbf95d8e4038972" -dependencies = [ - "crossbeam", - "dispose", - "futures-util", - "log", - "num_cpus", - "parking_lot 0.11.2", - "pin-project", - "thiserror", - "tokio", -] - [[package]] name = "tower" version = "0.4.13" diff --git a/Cargo.toml b/Cargo.toml index b0b42139d..81a886b97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,9 +131,6 @@ stretto = "0.8.4" thiserror = "1.0.31" tokio = "1.30.0" tokio-stream = "0.1.14" -topograph = { version = "0.4.0", features = [ - "tokio", -], git = "https://github.com/ray-kast/topograph", branch = "v0.4-wip" } tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" @@ -144,8 +141,8 @@ url = "2.3.1" vergen = "8.2.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" -yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin -yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22", default-features = false } # tag is geyser plugin [workspace.lints.clippy] diff --git a/README.md b/README.md index 87d87aa79..ce7af8e5e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,17 @@ super-fast querying and searching, as well as serves the merkle proofs needed to The API specification is located here https://github.com/metaplex-foundation/api-specifications This spec is what providers of this api must implement against. +#### GRPC-INGEST [/grpc-ingest/README.md](/grpc-ingest/README.md) + +This repo also houses the GRPC-INGEST component. This is a reimplementation of nft-ingester +This components separates grpc and ingester into two separate components. This is to allow for more flexibility in the future also increases the performance of the system. + +The two components are: + +- grpc2redis - This component listens to a gRPC stream (triton's Dragon's Mouth gRPC) for account and transaction updates and pushes them to a pipeline that flushes data to redis at regular intervals in real time. + +- ingester - This component listens to the redis stream and processes the data using program transformers. It also downloads token metadata json and stores them in a postgres db using sea-orm. + ### Infrastructure and Deployment Examples Along with the above rust binaries, this repo also maintains examples and best practice settings for running the entire infrastructure. diff --git a/docker-compose.yaml b/docker-compose.yaml index 8c44dc124..04098b2f6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -87,8 +87,17 @@ services: ports: - "6379:6379" db: - image: 'postgres:14' - command: [ "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ,"-c","max_connections=200" ] + image: "postgres:14" + command: + [ + "postgres", + "-c", + "log_statement=all", + "-c", + "log_destination=stderr", + "-c", + "max_connections=200", + ] ports: - 5432:5432 environment: @@ -119,9 +128,9 @@ services: - ./prometheus-config.yaml:/etc/prometheus/prometheus-config.yaml command: ["--config.file=/etc/prometheus/prometheus-config.yaml"] ports: - - "9091:9090" + - "9090:9090" extra_hosts: - "host.docker.internal:host-gateway" volumes: - grafana_data: { } - graphite_data: { } + grafana_data: {} + graphite_data: {} diff --git a/grpc-ingest/Cargo.toml b/grpc-ingest/Cargo.toml index 4036be057..9cbe506e2 100644 --- a/grpc-ingest/Cargo.toml +++ b/grpc-ingest/Cargo.toml @@ -52,7 +52,6 @@ tokio = { workspace = true, features = [ tracing = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } -topograph = { workspace = true, features = ["tokio"] } yellowstone-grpc-client = { workspace = true } yellowstone-grpc-proto = { workspace = true } yellowstone-grpc-tools = { workspace = true } diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md index d721d863b..011b2a062 100644 --- a/grpc-ingest/README.md +++ b/grpc-ingest/README.md @@ -1,33 +1,46 @@ -## Dev process +## Dev setup -### geyser gRPC source +> **Note:** Run these commands from the root of the project -Use [Triton One](https://triton.one/) provided endpoint or run own node with geyser plugin: https://github.com/rpcpool/yellowstone-grpc - -### Redis server +### Run redis, postgres and prometheus docker containers +```bash +docker compose up db redis prometheus ``` -redis-server + +### Seed the database and run migrations + +```bash +INIT_FILE_PATH=./init.sql sea migrate up --database-url=postgres://solana:solana@localhost:5432/solana ``` -### PostgreSQL server +### Config for grpc2redis [./config-grpc2redis.yml](./config-grpc2redis.yml) -Run: +### Run grpc2redis service -``` -docker run -it --rm -e POSTGRES_PASSWORD=solana -e POSTGRES_USER=solana -e POSTGRES_DB=solana -p 5432:5432 postgres +This service will listen to geyser gRPC account and transaction updates. It makes multiple subscriptions to the gRPC stream and filter the data based on the config. The data (vec of bytes) is pushed to a pipeline and then flushed to redis at regular intervals. + +> **Note:** Log level can be set to `info`, `debug`, `warn`, `error` + +```bash +RUST_LOG=info cargo run --bin das-grpc-ingest -- --config grpc-ingest/config-grpc2redis.yml grpc2redis ``` -Schema: +### Config for Ingester [./config-ingester.yml](./config-ingester.yml) -> Also note: The migration `m20230224_093722_performance_improvements` needs to be commented out of the migration lib.rs in order for the Sea ORM `Relations` to generate correctly. +### Run the Ingester service -``` -DATABASE_URL=postgres://solana:solana@localhost/solana INIT_FILE_PATH=init.sql cargo run -p migration --bin migration -- up -``` +This service performs many concurrent tasks -psql: +- Fetch account updates from redis and process them using using program_transformer +- Fetch transaction updates from redis and processe them +- Fetch snapshots from redis and process them +- download token metedata json and store them in postgres db +```bash + RUST_LOG=debug,sqlx=warn cargo run --bin das-grpc-ingest -- --config grpc-ingest/config-ingester.yml ingester ``` -PGPASSWORD=solana psql -h localhost -U solana -d solana -``` \ No newline at end of file + +### Metrics + +Both grpc2redis and ingester services expose prometheus metrics and can be accessed at `http://localhost:9090/metrics` diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.yml index 8ec202637..dcfa25bc7 100644 --- a/grpc-ingest/config-grpc2redis.yml +++ b/grpc-ingest/config-grpc2redis.yml @@ -1,26 +1,71 @@ -prometheus: 127.0.0.1:8873 -geyser_endpoint: http://127.0.0.1:10000 -x_token: null -commitment: finalized -max_concurrency: 5 -accounts: - stream: ACCOUNTS - stream_maxlen: 100_000_000 - filter: - owner: - - "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s" - - "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb" - - "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" - - "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL" - - "BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY" - - "CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d" -transactions: - stream: TRANSACTIONS - stream_maxlen: 10_000_000 - filter: - account_include: - - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY +# This file is used to configure the grpc2redis service. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8873 + +# gRPC server configuration (change x_token and endpoint to the correct ones) +geyser: + # endpoint of the dragonmouth stream + endpoint: http://127.0.0.1:10000 + # x-token of the dragonmouth stream + x_token: null + # transaction commitment level: finalized, confirmed, processed + commitment: finalized + # connection timeout in seconds + connection_timeout: 10 + # request timeout in seconds + timeout: 10 + +# gRPC subscription configuration (each representing a separate stream processed concurrently) +# check account and transaction filters here (https://github.com/rpcpool/yellowstone-grpc/blob/master/README.md) +subscriptions: + metadata: + stream: + # stream name + name: ACCOUNTS + # maximum length of the stream. + max_len: 100_000_000 + # maximum concurrency for processing the stream. + max_concurrency: 2 + # filter accounts by owner + filter: + accounts: + owner: + - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s + token: + stream: + name: ACCOUNTS + max_len: 100_000_000 + max_concurrency: 5 + filter: + accounts: + owner: + - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA + core: + stream: + name: ACCOUNTS + max_len: 100_000_000 + max_concurrency: 2 + filter: + accounts: + owner: + - CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d + bubblegum: + stream: + name: TRANSACTIONS + max_len: 100_000_000 + max_concurrency: 2 + # filter transactions by accounts_included + filter: + transactions: + account_include: + - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY + +# Redis configuration redis: + # redis connection url url: redis://localhost:6379 + # maximum size (in bytes) of the Redis pipeline before it is flushed. pipeline_max_size: 1_000 + # maximum idle time (in milliseconds) before the redis pipeline is flushed pipeline_max_idle_ms: 100_000 diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index a18ae2a54..d45d12e4e 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -1,16 +1,36 @@ -prometheus: 127.0.0.1:8875 +# This file is used to configure the ingester service. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8875 + +# redis connection url redis: redis://localhost:6379 + +# postgres configuration postgres: + # connection url url: postgres://solana:solana@localhost/solana + # min connections min_connections: 10 + # max connections max_connections: 50 + +# snapshots configuration for the ingester snapshots: + # stream name name: SNAPSHOTS + # maximum number of concurrent tasks for processing. max_concurrency: 10 + # maximum number of elements to return per redis stream batch_size: 100 + # maximum idle time (in milliseconds) before processing a batch xack_batch_max_idle_ms: 1_000 + # buffer size for redis acknowledgment messages xack_buffer_size: 10_000 + # maximum size of a batch for redis acknowledgment processing. xack_batch_max_size: 500 + +# accounts configuration for the ingester accounts: name: ACCOUNTS max_concurrency: 10 @@ -18,9 +38,15 @@ accounts: xack_batch_max_idle_ms: 1_000 xack_buffer_size: 10_000 xack_batch_max_size: 500 + +# transactions configuration for the ingester transactions: name: TRANSACTIONS + +# download metadata configuration for the ingester download_metadata: + # maximum number of attempts for downloading metadata JSON. max_attempts: 1 stream: + # stream name name: METADATA_JSON diff --git a/grpc-ingest/config-monitor.yml b/grpc-ingest/config-monitor.yml index 8b3278864..e2826a9e5 100644 --- a/grpc-ingest/config-monitor.yml +++ b/grpc-ingest/config-monitor.yml @@ -1,8 +1,20 @@ -prometheus: 127.0.0.1:8876 +# this file is used to configure the monitor service for bubblegum merkle trees. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8876 + +# rpc endpoint rpc: http://127.0.0.1:8899 + +# postgres configuration postgres: + # connection url url: postgres://solana:solana@localhost/solana + # min connections min_connections: 10 + # maximum allowed connections max_connections: 50 + +# bubblegum merkle tree configuration bubblegum: only_trees: null diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index cab28ad6c..e9e3ea644 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -1,7 +1,7 @@ use { anyhow::Context, serde::{de, Deserialize}, - std::{net::SocketAddr, path::Path, time::Duration}, + std::{collections::HashMap, net::SocketAddr, path::Path, time::Duration}, tokio::fs, yellowstone_grpc_tools::config::{ deserialize_usize_str, ConfigGrpcRequestAccounts, ConfigGrpcRequestCommitment, @@ -120,74 +120,81 @@ pub struct ConfigPrometheus { pub prometheus: Option, } -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigGrpc { +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGeyser { + pub endpoint: String, pub x_token: Option, - + #[serde(default = "ConfigGeyser::default_commitment")] pub commitment: ConfigGrpcRequestCommitment, - pub accounts: ConfigGrpcAccounts, - pub transactions: ConfigGrpcTransactions, - - pub geyser_endpoint: String, + #[serde(default = "ConfigGeyser::default_connection_timeout")] + pub connect_timeout: u64, + #[serde(default = "ConfigGeyser::default_timeout")] + pub timeout: u64, +} - pub redis: ConfigGrpcRedis, +impl ConfigGeyser { + pub const fn default_commitment() -> ConfigGrpcRequestCommitment { + ConfigGrpcRequestCommitment::Finalized + } - #[serde( - default = "ConfigGrpc::default_max_concurrency", - deserialize_with = "deserialize_usize_str" - )] - pub max_concurrency: usize, -} + pub const fn default_connection_timeout() -> u64 { + 10 + } -impl ConfigGrpc { - pub const fn default_max_concurrency() -> usize { + pub const fn default_timeout() -> u64 { 10 } } -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigGrpcAccounts { - pub stream: String, +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigStream { + pub name: String, #[serde( - default = "ConfigGrpcAccounts::default_stream_maxlen", + default = "ConfigStream::default_stream_maxlen", deserialize_with = "deserialize_usize_str" )] - pub stream_maxlen: usize, - #[serde(default = "ConfigGrpcAccounts::default_stream_data_key")] - pub stream_data_key: String, - - pub filter: ConfigGrpcRequestAccounts, + pub max_len: usize, + #[serde( + default = "ConfigStream::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, } -impl ConfigGrpcAccounts { +impl ConfigStream { pub const fn default_stream_maxlen() -> usize { - 100_000_000 + 10_000_000 } - pub fn default_stream_data_key() -> String { - REDIS_STREAM_DATA_KEY.to_owned() + pub const fn default_max_concurrency() -> usize { + 10 } } -#[derive(Debug, Clone, Deserialize)] -pub struct ConfigGrpcTransactions { - pub stream: String, - #[serde( - default = "ConfigGrpcTransactions::default_stream_maxlen", - deserialize_with = "deserialize_usize_str" - )] - pub stream_maxlen: usize, +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGrpcRequestFilter { + pub accounts: Option, + pub transactions: Option, +} - pub filter: ConfigGrpcRequestTransactions, +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigSubscription { + pub stream: ConfigStream, + pub filter: ConfigGrpcRequestFilter, } -impl ConfigGrpcTransactions { - pub const fn default_stream_maxlen() -> usize { - 10_000_000 - } +pub type ConfigGrpcSubscriptions = HashMap; + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGrpc { + pub geyser: ConfigGeyser, + + pub subscriptions: ConfigGrpcSubscriptions, + + pub redis: ConfigGrpcRedis, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Default)] pub struct ConfigGrpcRedis { pub url: String, #[serde( diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index f1a84b9d1..480975382 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -1,21 +1,19 @@ use { crate::{ - config::ConfigGrpc, + config::{ConfigGrpc, ConfigGrpcRequestFilter, ConfigSubscription}, prom::{grpc_tasks_total_dec, grpc_tasks_total_inc, redis_xadd_status_inc}, redis::TrackedPipeline, util::create_shutdown, }, anyhow::Context, - futures::{stream::StreamExt, SinkExt}, + futures::{ + stream::{FuturesUnordered, StreamExt}, + SinkExt, + }, redis::streams::StreamMaxlen, std::{collections::HashMap, sync::Arc, time::Duration}, tokio::{sync::Mutex, time::sleep}, - topograph::{ - executor::{Executor, Nonblock, Tokio}, - prelude::*, - AsyncHandler, - }, - tracing::{debug, warn}, + tracing::{debug, error, warn}, yellowstone_grpc_client::GeyserGrpcClient, yellowstone_grpc_proto::{ geyser::{SubscribeRequest, SubscribeRequestPing, SubscribeUpdate}, @@ -27,189 +25,297 @@ use { const PING_ID: i32 = 0; -enum GrpcJob { - FlushRedisPipe, - ProcessSubscribeUpdate(Box), +pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { + let redis_client = redis::Client::open(config.redis.url.clone())?; + let connection = redis_client.get_multiplexed_tokio_connection().await?; + + let mut shutdown = create_shutdown()?; + + let config = Arc::new(config); + + let subscriptions = config.subscriptions.clone(); + + let mut subscription_tasks = Vec::new(); + for (label, subscription_config) in subscriptions { + let subscription = Subscription { + label, + config: subscription_config, + }; + let task = SubscriptionTask::build() + .config(Arc::clone(&config)) + .connection(connection.clone()) + .subscription(subscription) + .start() + .await?; + + subscription_tasks.push(task); + } + + if let Some(signal) = shutdown.next().await { + warn!( + target: "grpc2redis", + action = "shutdown_signal_received", + message = "Shutdown signal received, waiting for spawned tasks to complete", + signal = ?signal + ); + } + + futures::future::join_all( + subscription_tasks + .into_iter() + .map(|task| task.stop()) + .collect::>(), + ) + .await + .into_iter() + .collect::>()?; + + Ok(()) } -#[derive(Clone)] -pub struct GrpcJobHandler { - connection: redis::aio::MultiplexedConnection, - config: Arc, - pipe: Arc>, +pub struct Subscription { + pub label: String, + pub config: ConfigSubscription, } -impl<'a> AsyncHandler>> - for GrpcJobHandler -{ - type Output = (); +#[derive(Default)] +pub struct SubscriptionTask { + pub config: Arc, + pub connection: Option, + pub subscription: Option, +} + +impl SubscriptionTask { + pub fn build() -> Self { + Self::default() + } - fn handle( - &self, - job: GrpcJob, - _handle: topograph::executor::Handle<'a, GrpcJob, Nonblock>, - ) -> impl futures::Future + Send + 'a { + pub fn config(mut self, config: Arc) -> Self { + self.config = config; + self + } + + pub fn subscription(mut self, subscription: Subscription) -> Self { + self.subscription = Some(subscription); + self + } + + pub fn connection(mut self, connection: redis::aio::MultiplexedConnection) -> Self { + self.connection = Some(connection); + self + } + + pub async fn start(mut self) -> anyhow::Result { let config = Arc::clone(&self.config); - let connection = self.connection.clone(); - let pipe = Arc::clone(&self.pipe); + let connection = self + .connection + .take() + .expect("Redis Connection is required"); - grpc_tasks_total_inc(); + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let subscription = self.subscription.take().expect("Subscription is required"); + let label = subscription.label.clone(); + let subscription_config = Arc::new(subscription.config); + let connection = connection.clone(); - async move { - match job { - GrpcJob::FlushRedisPipe => { - let mut pipe = pipe.lock().await; - let mut connection = connection; + let ConfigSubscription { stream, filter } = subscription_config.as_ref().clone(); - let flush = pipe.flush(&mut connection).await; + let stream_config = Arc::new(stream.clone()); + let mut req_accounts = HashMap::with_capacity(1); + let mut req_transactions = HashMap::with_capacity(1); - let status = flush.as_ref().map(|_| ()).map_err(|_| ()); - let counts = flush.as_ref().unwrap_or_else(|counts| counts); + let ConfigGrpcRequestFilter { + accounts, + transactions, + } = filter; - for (stream, count) in counts.iter() { - debug!(target: "grpc2redis", action = "flush_redis_pipe", stream = ?stream, status = ?status, count = ?count); - redis_xadd_status_inc(stream, status, *count); - } - } - GrpcJob::ProcessSubscribeUpdate(update) => { - let accounts_stream = config.accounts.stream.clone(); - let accounts_stream_maxlen = config.accounts.stream_maxlen; - let transactions_stream = config.transactions.stream.clone(); - let transactions_stream_maxlen = config.transactions.stream_maxlen; - - let SubscribeUpdate { update_oneof, .. } = *update; - - let mut pipe = pipe.lock().await; - - if let Some(update) = update_oneof { - match update { - UpdateOneof::Account(account) => { - pipe.xadd_maxlen( - &accounts_stream, - StreamMaxlen::Approx(accounts_stream_maxlen), - "*", - account.encode_to_vec(), - ); - debug!(target: "grpc2redis", action = "process_account_update", stream = ?accounts_stream, maxlen = ?accounts_stream_maxlen); - } - UpdateOneof::Transaction(transaction) => { - pipe.xadd_maxlen( - &transactions_stream, - StreamMaxlen::Approx(transactions_stream_maxlen), - "*", - transaction.encode_to_vec(), - ); - debug!(target: "grpc2redis", action = "process_transaction_update", stream = ?transactions_stream, maxlen = ?transactions_stream_maxlen); - } - _ => { - warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant") - } - } - } - } - } + if let Some(accounts) = accounts { + req_accounts.insert(label.clone(), accounts.to_proto()); + } - grpc_tasks_total_dec(); + if let Some(transactions) = transactions { + req_transactions.insert(label.clone(), transactions.to_proto()); } - } -} -pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { - let redis_client = redis::Client::open(config.redis.url.clone())?; - let config = Arc::new(config); - let connection = redis_client.get_multiplexed_tokio_connection().await?; + let request = SubscribeRequest { + accounts: req_accounts, + transactions: req_transactions, + ..Default::default() + }; - let mut shutdown = create_shutdown()?; + let pipe = Arc::new(Mutex::new(TrackedPipeline::default())); + let mut tasks = FuturesUnordered::new(); - let pipe = Arc::new(Mutex::new(TrackedPipeline::default())); - - let mut accounts = HashMap::with_capacity(1); - let mut transactions = HashMap::with_capacity(1); - - accounts.insert("das".to_string(), config.accounts.filter.clone().to_proto()); - transactions.insert( - "das".to_string(), - config.transactions.filter.clone().to_proto(), - ); - - let request = SubscribeRequest { - accounts, - transactions, - ..Default::default() - }; - - let mut dragon_mouth_client = - GeyserGrpcClient::build_from_shared(config.geyser_endpoint.clone())? - .x_token(config.x_token.clone())? - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(10)) - .connect() - .await - .context("failed to connect to gRPC")?; - - let (mut subscribe_tx, stream) = dragon_mouth_client - .subscribe_with_request(Some(request)) - .await?; - - tokio::pin!(stream); - - let exec = Executor::builder(Nonblock(Tokio)) - .max_concurrency(Some(config.max_concurrency)) - .build_async(GrpcJobHandler { - config: Arc::clone(&config), - connection: connection.clone(), - pipe: Arc::clone(&pipe), - })?; - - let deadline_config = Arc::clone(&config); - - loop { - tokio::select! { - _ = sleep(deadline_config.redis.pipeline_max_idle) => { - exec.push(GrpcJob::FlushRedisPipe); - } - Some(Ok(msg)) = stream.next() => { - match msg.update_oneof { - Some(UpdateOneof::Account(_)) | Some(UpdateOneof::Transaction(_)) => { - exec.push(GrpcJob::ProcessSubscribeUpdate(Box::new(msg))); + let mut dragon_mouth_client = + GeyserGrpcClient::build_from_shared(config.geyser.endpoint.clone())? + .x_token(config.geyser.x_token.clone())? + .connect_timeout(Duration::from_secs(config.geyser.connect_timeout)) + .timeout(Duration::from_secs(config.geyser.timeout)) + .connect() + .await + .context("failed to connect to gRPC")?; + + let (mut subscribe_tx, stream) = dragon_mouth_client + .subscribe_with_request(Some(request)) + .await?; + + let deadline_config = Arc::clone(&config); + + let control = tokio::spawn({ + async move { + tokio::pin!(stream); + + let (flush_tx, mut flush_rx) = tokio::sync::mpsc::channel::<()>(1); + + let flush_handle = tokio::spawn({ + let pipe = Arc::clone(&pipe); + let stream_config = Arc::clone(&stream_config); + let label = label.clone(); + let mut connection = connection.clone(); + + async move { + while (flush_rx.recv().await).is_some() { + let mut pipe = pipe.lock().await; + let flush = pipe.flush(&mut connection).await; + + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let count = flush.as_ref().unwrap_or_else(|count| count); + + debug!(target: "grpc2redis", action = "flush_redis_pipe", stream = ?stream_config.name, status = ?status, count = ?count); + redis_xadd_status_inc(&stream_config.name, &label, status, *count); + } } - Some(UpdateOneof::Ping(_)) => { - let ping = subscribe_tx - .send(SubscribeRequest { - ping: Some(SubscribeRequestPing { id: PING_ID }), - ..Default::default() - }) - .await; - - match ping { - Ok(_) => { - debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID) - } - Err(err) => { - warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID) + }); + + loop { + tokio::select! { + _ = sleep(deadline_config.redis.pipeline_max_idle) => { + let _ = flush_tx.send(()).await; + } + Some(Ok(msg)) = stream.next() => { + match msg.update_oneof { + Some(UpdateOneof::Account(_)) | Some(UpdateOneof::Transaction(_)) => { + if tasks.len() >= stream_config.max_concurrency { + tasks.next().await; + } + grpc_tasks_total_inc(&label, &stream_config.name); + + tasks.push(tokio::spawn({ + let pipe = Arc::clone(&pipe); + let label = label.clone(); + let stream_config = Arc::clone(&stream_config); + + async move { + let stream = stream_config.name.clone(); + let stream_maxlen = stream_config.max_len; + + let SubscribeUpdate { update_oneof, .. } = msg; + + let mut pipe = pipe.lock().await; + + if let Some(update) = update_oneof { + match update { + UpdateOneof::Account(account) => { + pipe.xadd_maxlen( + &stream.to_string(), + StreamMaxlen::Approx(stream_maxlen), + "*", + account.encode_to_vec(), + ); + debug!(target: "grpc2redis", action = "process_account_update",label = ?label, stream = ?stream, maxlen = ?stream_maxlen); + } + + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &stream.to_string(), + StreamMaxlen::Approx(stream_maxlen), + "*", + transaction.encode_to_vec(), + ); + debug!(target: "grpc2redis", action = "process_transaction_update",label = ?label, stream = ?stream, maxlen = ?stream_maxlen); + } + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant",label = ?label, message = "Unknown update variant") + } + } + } + + grpc_tasks_total_dec(&label, &stream_config.name); + } + } + )) + } + Some(UpdateOneof::Ping(_)) => { + let ping = subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: PING_ID }), + ..Default::default() + }) + .await; + + match ping { + Ok(_) => { + debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID) + } + Err(err) => { + warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID) + } + } + } + Some(UpdateOneof::Pong(pong)) => { + if pong.id == PING_ID { + debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); + } else { + warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); + } + } + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant", ?msg.update_oneof) + } } } - } - Some(UpdateOneof::Pong(pong)) => { - if pong.id == PING_ID { - debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); - } else { - warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); + _ = &mut shutdown_rx => { + debug!(target: "grpc2redis", action = "shutdown_signal_received", message = "Shutdown signal received, stopping subscription task", ?label); + break; } } - _ => { - warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant", ?msg.update_oneof) - } + } + + while (tasks.next().await).is_some() {} + + if let Err(err) = flush_tx.send(()).await { + error!(target: "grpc2redis", action = "flush_send_failed", message = "Failed to send flush signal", ?err); + } + + drop(flush_tx); + + if let Err(err) = flush_handle.await { + error!(target: "grpc2redis", action = "flush_failed", message = "Failed to flush", ?err); } } - _ = shutdown.next() => { - exec.push(GrpcJob::FlushRedisPipe); - break; - } - } + }); + + Ok(SubscriptionTaskStop { + shutdown_tx, + control, + }) } +} - exec.join_async().await; +#[derive(Debug)] +pub struct SubscriptionTaskStop { + pub shutdown_tx: tokio::sync::oneshot::Sender<()>, + pub control: tokio::task::JoinHandle<()>, +} - Ok(()) +impl SubscriptionTaskStop { + pub async fn stop(self) -> anyhow::Result<()> { + self.shutdown_tx + .send(()) + .map_err(|_| anyhow::anyhow!("Failed to send shutdown signal"))?; + + self.control.await?; + + Ok(()) + } } diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index bc46b79f0..710415d5f 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -45,7 +45,7 @@ fn download_metadata_notifier_v2( let status = xadd.map(|_| ()).map_err(|_| ()); - redis_xadd_status_inc(&stream, status, 1); + redis_xadd_status_inc(&stream, "metadata_notifier",status, 1); Ok(()) }) @@ -133,7 +133,9 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { snapshots.stop(), download_metadatas.stop(), ]) - .await; + .await + .into_iter() + .collect::>()?; report.abort(); diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index a82022796..9819d9085 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -28,7 +28,7 @@ lazy_static::lazy_static! { static ref REDIS_XADD_STATUS_COUNT: IntCounterVec = IntCounterVec::new( Opts::new("redis_xadd_status_count", "Status of messages sent to Redis stream"), - &["stream", "status"] + &["stream", "label", "status"] ).unwrap(); static ref REDIS_XREAD_COUNT: IntCounterVec = IntCounterVec::new( @@ -71,9 +71,10 @@ lazy_static::lazy_static! { static ref GRPC_TASKS: IntGaugeVec = IntGaugeVec::new( Opts::new("grpc_tasks", "Number of tasks spawned for writing grpc messages to redis "), - &[] + &["label","stream"] ).unwrap(); + static ref BUBBLEGUM_TREE_TOTAL_LEAVES: IntGaugeVec = IntGaugeVec::new( Opts::new("bubblegum_tree_total_leaves", "Total number of leaves in the bubblegum tree"), &["tree"] @@ -122,11 +123,6 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(INGEST_TASKS); register!(ACK_TASKS); register!(GRPC_TASKS); - register!(BUBBLEGUM_TREE_TOTAL_LEAVES); - register!(BUBBLEGUM_TREE_INCORRECT_PROOFS); - register!(BUBBLEGUM_TREE_NOT_FOUND_PROOFS); - register!(BUBBLEGUM_TREE_CORRECT_PROOFS); - register!(BUBBLEGUM_TREE_CORRUPT_PROOFS); VERSION_INFO_METRIC .with_label_values(&[ @@ -151,7 +147,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { })) }); let server = Server::try_bind(&address)?.serve(make_service); - info!("prometheus server started: {address:?}"); + info!("prometheus server started: http://{address:?}/metrics"); tokio::spawn(async move { if let Err(error) = server.await { error!("prometheus server failed: {error:?}"); @@ -184,9 +180,13 @@ pub fn redis_xlen_set(stream: &str, len: usize) { .set(len as i64); } -pub fn redis_xadd_status_inc(stream: &str, status: Result<(), ()>, delta: usize) { +pub fn redis_xadd_status_inc(stream: &str, label: &str, status: Result<(), ()>, delta: usize) { REDIS_XADD_STATUS_COUNT - .with_label_values(&[stream, if status.is_ok() { "success" } else { "failed" }]) + .with_label_values(&[ + stream, + label, + if status.is_ok() { "success" } else { "failed" }, + ]) .inc_by(delta as u64); } @@ -233,12 +233,12 @@ pub fn ack_tasks_total_dec(stream: &str) { ACK_TASKS.with_label_values(&[stream]).dec() } -pub fn grpc_tasks_total_inc() { - GRPC_TASKS.with_label_values(&[]).inc() +pub fn grpc_tasks_total_inc(label: &str, stream: &str) { + GRPC_TASKS.with_label_values(&[label, stream]).inc() } -pub fn grpc_tasks_total_dec() { - GRPC_TASKS.with_label_values(&[]).dec() +pub fn grpc_tasks_total_dec(label: &str, stream: &str) { + GRPC_TASKS.with_label_values(&[label, stream]).dec() } #[derive(Debug, Clone, Copy)] diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 1e9d05783..7fe5e2a3c 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -578,22 +578,21 @@ impl IngestStream { } } +#[derive(Clone)] pub struct TrackedPipeline { pipeline: redis::Pipeline, - counts: HashMap, + count: usize, } impl Default for TrackedPipeline { fn default() -> Self { Self { pipeline: redis::pipe(), - counts: HashMap::new(), + count: 0, } } } -type TrackedStreamCounts = HashMap; - impl TrackedPipeline { pub fn xadd_maxlen(&mut self, key: &str, maxlen: StreamMaxlen, id: F, field: V) where @@ -602,21 +601,18 @@ impl TrackedPipeline { { self.pipeline .xadd_maxlen(key, maxlen, id, &[(REDIS_STREAM_DATA_KEY, field)]); - *self.counts.entry(key.to_string()).or_insert(0) += 1; + self.count += 1; } - pub async fn flush( - &mut self, - connection: &mut MultiplexedConnection, - ) -> Result { + pub async fn flush(&mut self, connection: &mut MultiplexedConnection) -> Result { let result: RedisResult = self.pipeline.atomic().query_async(connection).await; - let counts = self.counts.clone(); - self.counts.clear(); + let count = self.count; + self.count = 0; self.pipeline.clear(); match result { - Ok(_) => Ok(counts), - Err(_) => Err(counts), + Ok(_) => Ok(count), + Err(_) => Err(count), } } } diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 000000000..d81cc4a24 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,41 @@ +### DAS Ops + +DAS Ops is a collection of operational tools and scripts for managing and maintaining the Digital Asset RPC infrastructure. + +> **Note:** Run these commands from the root of the project + +### Setup + +```bash +sudo docker compose up db +``` + +### Running the cli + +```bash +cargo run --bin das-ops -- --help +``` + +#### Required Args + +- `--solana-rpc-url` - RPC URL of the Solana cluster +- `--database-url` - URL of the Postgres database (if using Docker: `postgres://solana:solana@localhost:5432/solana`) + +### Commands + +- `account` : Account related operations + + #### Subcommands + + - `program ` command is used to backfill the index against on-chain accounts owned by a program + + - `single ` command is used to backfill the index against a single account + + - `nft ` command is used to backfill the index against an NFT mint, token metadata, and token account + +- `bubblegum` : Bubblegum program related operations + + #### Subcommands + + - `backfill` command is used to cross-reference the index against on-chain accounts. It crawls through trees and backfills any missed tree transactions. + - `replay ` command is used to replay the Bubblegum program transactions for a given tree address and parse all the cNFT instructions diff --git a/ops/src/account/cmd.rs b/ops/src/account/cmd.rs index 9861cd8fe..c36fdd22e 100644 --- a/ops/src/account/cmd.rs +++ b/ops/src/account/cmd.rs @@ -10,7 +10,7 @@ pub enum Commands { /// The 'single' command is used to backfill the index against a single account. #[clap(name = "single")] Single(single::Args), - + /// The 'nft' command is used to backfill the index against an NFT mint, token metadata, and token account. #[clap(name = "nft")] Nft(nft::Args), } From f311e149f245b65fb216bd7e8568538c68a2a9ec Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Fri, 8 Nov 2024 21:23:43 +0100 Subject: [PATCH 62/72] Report proofs as they come (#178) --- bubblegum/src/lib.rs | 20 ++++++++++++-------- bubblegum/src/verify.rs | 17 +++++++++++------ grpc-ingest/src/monitor.rs | 15 +++++++++------ grpc-ingest/src/prom.rs | 8 ++++++++ ops/src/bubblegum/verify.rs | 4 ++-- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/bubblegum/src/lib.rs b/bubblegum/src/lib.rs index f1dc031c6..463526337 100644 --- a/bubblegum/src/lib.rs +++ b/bubblegum/src/lib.rs @@ -165,20 +165,24 @@ pub struct VerifyArgs { pub async fn verify_bubblegum( context: BubblegumContext, args: VerifyArgs, -) -> Result> { +) -> Result> { let trees = if let Some(ref only_trees) = args.only_trees { TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? } else { TreeResponse::all(&context.solana_rpc).await? }; - let mut reports = Vec::new(); + let (sender, receiver) = tokio::sync::mpsc::channel(trees.len()); - for tree in trees { - let report = verify::check(context.clone(), tree, args.max_concurrency).await?; - - reports.push(report); - } + tokio::spawn(async move { + for tree in trees { + if let Ok(report) = verify::check(context.clone(), tree, args.max_concurrency).await { + if sender.send(report).await.is_err() { + error!("Failed to send report"); + } + } + } + }); - Ok(reports) + Ok(receiver) } diff --git a/bubblegum/src/verify.rs b/bubblegum/src/verify.rs index fae6051d6..e99972ce0 100644 --- a/bubblegum/src/verify.rs +++ b/bubblegum/src/verify.rs @@ -13,7 +13,7 @@ use spl_account_compression::concurrent_tree_wrapper::ProveLeafArgs; use std::fmt; use std::sync::Arc; use tokio::sync::Mutex; -use tracing::error; +use tracing::debug; trait TryFromAssetProof { fn try_from_asset_proof(proof: AssetProof) -> Result @@ -89,6 +89,7 @@ pub struct ProofReport { pub corrupt_proofs: usize, } +#[derive(Debug)] enum ProofResult { Correct, Incorrect, @@ -152,17 +153,21 @@ pub async fn check( if let Ok(proof_result) = proof_lookup { let mut report = report.lock().await; + match proof_result { ProofResult::Correct => report.correct_proofs += 1, ProofResult::Incorrect => report.incorrect_proofs += 1, ProofResult::NotFound => report.not_found_proofs += 1, ProofResult::Corrupt => report.corrupt_proofs += 1, } - if let ProofResult::Incorrect | ProofResult::NotFound | ProofResult::Corrupt = - proof_result - { - error!(tree = %tree_pubkey, leaf_index = i, asset = %asset, "{}", proof_result); - } + + debug!( + tree = %tree_pubkey, + leaf_index = i, + asset = %asset, + result = ?proof_result, + "Proof result for asset" + ); } })); } diff --git a/grpc-ingest/src/monitor.rs b/grpc-ingest/src/monitor.rs index ac0c687ec..7c7e277f8 100644 --- a/grpc-ingest/src/monitor.rs +++ b/grpc-ingest/src/monitor.rs @@ -4,7 +4,7 @@ use crate::{config::ConfigMonitor, prom::update_tree_proof_report}; use das_bubblegum::{verify_bubblegum, BubblegumContext, VerifyArgs}; use das_core::{Rpc, SolanaRpcArgs}; use futures::stream::StreamExt; -use tracing::error; +use tracing::{error, info}; pub async fn run(config: ConfigMonitor) -> anyhow::Result<()> { let mut shutdown = create_shutdown()?; @@ -22,20 +22,23 @@ pub async fn run(config: ConfigMonitor) -> anyhow::Result<()> { }; match verify_bubblegum(bubblegum_context, verify_args).await { - Ok(reports) => { - for report in reports { + Ok(mut reports_receiver) => { + while let Some(report) = reports_receiver.recv().await { + info!( + report = ?report, + ); update_tree_proof_report(&report); } + + tokio::time::sleep(tokio::time::Duration::from_secs(600)).await; } Err(e) => { error!( - message = "Error verifying bubblegum", + message = "Error proof report recv", error = ?e ); } } - - tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; } }); diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 9819d9085..44f9557d7 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -111,6 +111,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { .expect("collector can't be registered"); }; } + register!(VERSION_INFO_METRIC); register!(REDIS_STREAM_LENGTH); register!(REDIS_XADD_STATUS_COUNT); @@ -123,6 +124,11 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(INGEST_TASKS); register!(ACK_TASKS); register!(GRPC_TASKS); + register!(BUBBLEGUM_TREE_TOTAL_LEAVES); + register!(BUBBLEGUM_TREE_INCORRECT_PROOFS); + register!(BUBBLEGUM_TREE_NOT_FOUND_PROOFS); + register!(BUBBLEGUM_TREE_CORRECT_PROOFS); + register!(BUBBLEGUM_TREE_CORRUPT_PROOFS); VERSION_INFO_METRIC .with_label_values(&[ @@ -146,8 +152,10 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { Ok::<_, hyper::Error>(response) })) }); + let server = Server::try_bind(&address)?.serve(make_service); info!("prometheus server started: http://{address:?}/metrics"); + tokio::spawn(async move { if let Err(error) = server.await { error!("prometheus server failed: {error:?}"); diff --git a/ops/src/bubblegum/verify.rs b/ops/src/bubblegum/verify.rs index 947d2b3ad..46cb97cfc 100644 --- a/ops/src/bubblegum/verify.rs +++ b/ops/src/bubblegum/verify.rs @@ -25,9 +25,9 @@ pub async fn run(config: Args) -> Result<()> { let solana_rpc = Rpc::from_config(&config.solana); let context = BubblegumContext::new(database_pool, solana_rpc); - let reports = verify_bubblegum(context, config.verify_bubblegum).await?; + let mut reports = verify_bubblegum(context, config.verify_bubblegum).await?; - for report in reports { + while let Some(report) = reports.recv().await { info!( "Tree: {}, Total Leaves: {}, Incorrect Proofs: {}, Not Found Proofs: {}, Correct Proofs: {}", report.tree_pubkey, From f8bdef2ddb85a66788d5b8a3467e4600c559ef2d Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 11 Nov 2024 13:15:38 +0100 Subject: [PATCH 63/72] Buffer redis messages (#179) --- grpc-ingest/config-ingester.yml | 2 + grpc-ingest/src/config.rs | 10 +++ grpc-ingest/src/redis.rs | 118 ++++++++++++++++++++------------ prometheus-config.yaml | 7 +- 4 files changed, 93 insertions(+), 44 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index d45d12e4e..e0e2850e1 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -29,6 +29,8 @@ snapshots: xack_buffer_size: 10_000 # maximum size of a batch for redis acknowledgment processing. xack_batch_max_size: 500 + # maximum number of redis messages to keep to buffer. + messages_buffer_size: 200 # accounts configuration for the ingester accounts: diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index e9e3ea644..b34ba8fe8 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -66,12 +66,22 @@ pub struct ConfigIngestStream { deserialize_with = "deserialize_usize_str" )] pub xack_buffer_size: usize, + #[serde( + default = "ConfigIngestStream::default_message_buffer_size", + deserialize_with = "deserialize_usize_str" + )] + pub message_buffer_size: usize, } impl ConfigIngestStream { pub const fn default_xack_buffer_size() -> usize { 1_000 } + + pub const fn default_message_buffer_size() -> usize { + 100 + } + pub const fn default_max_concurrency() -> usize { 2 } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 7fe5e2a3c..0d9e093c5 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -299,7 +299,6 @@ impl Acknowledge { let count = ids.len(); - ack_tasks_total_inc(&config.name); match redis::pipe() .atomic() .xack(&config.name, &config.group, &ids) @@ -403,6 +402,68 @@ impl IngestStream { let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); let (ack_shutdown_tx, mut ack_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let (msg_tx, mut msg_rx) = + tokio::sync::mpsc::channel::>(config.message_buffer_size); + let (msg_shutdown_tx, mut msg_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + + let config_messages = Arc::clone(&config); + + let messages = tokio::spawn(async move { + let mut tasks = FuturesUnordered::new(); + let config = Arc::clone(&config_messages); + let handler = handler.clone(); + + loop { + tokio::select! { + Some(ids) = msg_rx.recv() => { + for StreamId { id, map } in ids { + if tasks.len() >= config.max_concurrency { + tasks.next().await; + } + + let handler = handler.clone(); + let ack_tx = ack_tx.clone(); + let config = Arc::clone(&config); + + ingest_tasks_total_inc(&config.name); + + tasks.push(tokio::spawn(async move { + let result = handler.handle(map).await.map_err(IngestMessageError::into); + + match result { + Ok(()) => { + program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success); + } + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); + program_transformer_task_status_inc(e.into()); + } + } + + if let Err(e) = ack_tx.send(id).await { + error!(target: "ingest_stream", "action=send_ack stream={} error={:?}", &config.name, e); + } + + ingest_tasks_total_dec(&config.name); + })); + } + } + _ = &mut msg_shutdown_rx => { + break; + } + } + } + + while (tasks.next().await).is_some() {} + }); + let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); @@ -426,6 +487,7 @@ impl IngestStream { let ids = std::mem::take(&mut pending); let handler = Arc::clone(&handler); + tasks.push(tokio::spawn(async move { handler.handle(ids).await; })); @@ -439,6 +501,9 @@ impl IngestStream { } let ids = std::mem::take(&mut pending); let handler = Arc::clone(&handler); + + ack_tasks_total_inc(&config.name); + tasks.push(tokio::spawn(async move { handler.handle(ids).await; })); @@ -482,11 +547,9 @@ impl IngestStream { let control = tokio::spawn({ let mut connection = connection.clone(); - let mut tasks = FuturesUnordered::new(); async move { let config = Arc::clone(&config); - let handler = handler.clone(); debug!(target: "ingest_stream", "action=read_stream_start stream={}", config.name); @@ -495,7 +558,13 @@ impl IngestStream { tokio::select! { _ = &mut shutdown_rx => { - while (tasks.next().await).is_some() {} + if let Err(e) = msg_shutdown_tx.send(()) { + error!(target: "ingest_stream", "action=msg_shutdown stream={} error={:?}", &config.name, e); + } + + if let Err(e) = messages.await { + error!(target: "ingest_stream", "action=await_messages stream={} error={:?}", &config.name, e); + } if let Err(e) = ack_shutdown_tx.send(()) { error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", &config.name, e); @@ -517,45 +586,8 @@ impl IngestStream { redis_xread_inc(&config.name, count); - for StreamId { id, map } in ids { - if tasks.len() >= config.max_concurrency { - tasks.next().await; - } - - let handler = handler.clone(); - let ack_tx = ack_tx.clone(); - let config = Arc::clone(&config); - - ingest_tasks_total_inc(&config.name); - - tasks.push(tokio::spawn(async move { - let result = handler.handle(map).await.map_err(IngestMessageError::into); - - match result { - Ok(()) => { - program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success); - } - Err(IngestMessageError::RedisStreamMessage(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::DownloadMetadataJson(e)) => { - program_transformer_task_status_inc(e.into()); - } - Err(IngestMessageError::ProgramTransformer(e)) => { - error!("Failed to process message: {:?}", e); - - program_transformer_task_status_inc(e.into()); - } - } - - ingest_tasks_total_dec(&config.name); - - if let Err(e) = ack_tx.send(id).await { - error!(target: "ingest_stream", "action=send_ack stream={} error={:?}", &config.name, e); - } - })); + if let Err(e) = msg_tx.send(ids).await { + error!(target: "ingest_stream", "action=send_ids stream={} error={:?}", &config.name, e); } } } diff --git a/prometheus-config.yaml b/prometheus-config.yaml index 848890eb6..3f975ab85 100644 --- a/prometheus-config.yaml +++ b/prometheus-config.yaml @@ -6,4 +6,9 @@ scrape_configs: - job_name: "prometheus" honor_labels: true static_configs: - - targets: ["host.docker.internal:8873", "host.docker.internal:8875"] + - targets: + [ + "host.docker.internal:8873", + "host.docker.internal:8875", + "host.docker.internal:8876", + ] From 776ae0a6b903d2f67b3e86224edc3fcb3efe3ab9 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 11 Nov 2024 16:46:47 +0100 Subject: [PATCH 64/72] Fix tracking ack tasks --- grpc-ingest/src/redis.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 0d9e093c5..abd36ef29 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -312,8 +312,6 @@ impl Acknowledge { "action=acknowledge_and_delete stream={} response={:?} expected={:?}", config.name, response, count ); - - redis_xack_inc(&config.name, count); } Err(e) => { error!( @@ -488,6 +486,8 @@ impl IngestStream { let handler = Arc::clone(&handler); + ack_tasks_total_inc(&config.name); + tasks.push(tokio::spawn(async move { handler.handle(ids).await; })); @@ -534,13 +534,14 @@ impl IngestStream { let config = Arc::clone(&config); loop { - sleep(Duration::from_millis(100)).await; let connection = connection.clone(); let labels = labels.clone(); if let Err(e) = report_xlen(connection, labels).await { error!(target: "ingest_stream", "action=report_xlen stream={} error={:?}", &config.name, e); } + + sleep(Duration::from_millis(100)).await; } } }); From 151168e67a695f7bf0f166f5b0ad2b78c3388766 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 11 Nov 2024 18:41:31 +0100 Subject: [PATCH 65/72] fix: add back ack tasks total --- grpc-ingest/src/redis.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index abd36ef29..95eda19b7 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -312,6 +312,8 @@ impl Acknowledge { "action=acknowledge_and_delete stream={} response={:?} expected={:?}", config.name, response, count ); + + redis_xack_inc(&config.name, count); } Err(e) => { error!( From ae1190723c4c93727c5b70397da82bd83a154ed5 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Tue, 12 Nov 2024 17:56:30 +0100 Subject: [PATCH 66/72] recreate consumer but not groups (#181) --- grpc-ingest/config-ingester.yml | 3 +- grpc-ingest/src/redis.rs | 49 ++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.yml index e0e2850e1..37815047e 100644 --- a/grpc-ingest/config-ingester.yml +++ b/grpc-ingest/config-ingester.yml @@ -31,6 +31,8 @@ snapshots: xack_batch_max_size: 500 # maximum number of redis messages to keep to buffer. messages_buffer_size: 200 + # redis stream consumer name must be unique per ingester process + consumer: consumer # accounts configuration for the ingester accounts: @@ -50,5 +52,4 @@ download_metadata: # maximum number of attempts for downloading metadata JSON. max_attempts: 1 stream: - # stream name name: METADATA_JSON diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 95eda19b7..9903bb5f4 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -380,24 +380,23 @@ impl IngestStream { let mut connection = self.connection.take().expect("Connection is required"); let handler = self.handler.take().expect("Handler is required"); - if let Err(e) = xgroup_delete(&mut connection, &config.name, &config.group).await { - error!(target: "ingest_stream", "action=xgroup_delete stream={} error={:?}", config.name, e); - } else { - debug!(target: "ingest_stream", "action=xgroup_delete stream={} group={}", config.name, config.group); - } + xgroup_create(&mut connection, &config.name, &config.group).await?; - if let Err(e) = xgroup_create( + xgroup_delete_consumer( &mut connection, &config.name, &config.group, &config.consumer, ) - .await - { - error!(target: "ingest_stream", "action=xgroup_create stream={} error={:?}", config.name, e); - } else { - debug!(target: "ingest_stream", "action=xgroup_create stream={} group={} consumer={}", config.name, config.group, config.consumer); - } + .await?; + + xgroup_create_consumer( + &mut connection, + &config.name, + &config.group, + &config.consumer, + ) + .await?; let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); let (ack_shutdown_tx, mut ack_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); @@ -673,7 +672,6 @@ pub async fn xgroup_create( connection: &mut C, name: &str, group: &str, - consumer: &str, ) -> anyhow::Result<()> { let result: RedisResult = connection.xgroup_create_mkstream(name, group, "0").await; if let Err(error) = result { @@ -685,27 +683,40 @@ pub async fn xgroup_create( } } - // XGROUP CREATECONSUMER key group consumer - redis::cmd("XGROUP") + Ok(()) +} + +pub async fn xgroup_create_consumer( + connection: &mut C, + name: &str, + group: &str, + consumer: &str, +) -> anyhow::Result<()> { + let result: RedisResult = redis::cmd("XGROUP") .arg("CREATECONSUMER") .arg(name) .arg(group) .arg(consumer) .query_async(connection) - .await?; + .await; - Ok(()) + match result { + Ok(_) => Ok(()), + Err(error) => Err(error.into()), + } } -pub async fn xgroup_delete( +pub async fn xgroup_delete_consumer( connection: &mut C, name: &str, group: &str, + consumer: &str, ) -> anyhow::Result<()> { let result: RedisResult = redis::cmd("XGROUP") - .arg("DESTROY") + .arg("DELCONSUMER") .arg(name) .arg(group) + .arg(consumer) .query_async(connection) .await; From 80d61dc75200fe76d89c59aa747f582d144d5b5c Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 14 Nov 2024 16:31:38 +0100 Subject: [PATCH 67/72] Time Ingest jobs (#182) * time the program transformer tasks * add consumer labels to metrics --- grpc-ingest/src/prom.rs | 67 ++++++++++++++++++++++++---------------- grpc-ingest/src/redis.rs | 28 ++++++++++------- 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 44f9557d7..1bb3a125a 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -8,7 +8,10 @@ use { Body, Request, Response, Server, StatusCode, }, program_transformers::error::ProgramTransformerError, - prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder}, + prometheus::{ + HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, Opts, Registry, + TextEncoder, + }, std::{net::SocketAddr, sync::Once}, tracing::{error, info}, }; @@ -31,14 +34,14 @@ lazy_static::lazy_static! { &["stream", "label", "status"] ).unwrap(); - static ref REDIS_XREAD_COUNT: IntCounterVec = IntCounterVec::new( + static ref REDIS_XREAD_COUNT: IntCounterVec = IntCounterVec::new( Opts::new("redis_xread_count", "Count of messages seen"), - &["stream"] + &["stream", "consumer"] ).unwrap(); static ref REDIS_XACK_COUNT: IntCounterVec = IntCounterVec::new( Opts::new("redis_xack_count", "Total number of processed messages"), - &["stream"] + &["stream", "consumer"] ).unwrap(); static ref PGPOOL_CONNECTIONS: IntGaugeVec = IntGaugeVec::new( @@ -46,13 +49,14 @@ lazy_static::lazy_static! { &["kind"] ).unwrap(); - static ref PROGRAM_TRANSFORMER_TASKS: IntGauge = IntGauge::new( - "program_transformer_tasks", "Number of tasks spawned for program transform" - ).unwrap(); - static ref PROGRAM_TRANSFORMER_TASK_STATUS_COUNT: IntCounterVec = IntCounterVec::new( Opts::new("program_transformer_task_status_count", "Status of processed messages"), - &["status"], + &["stream", "consumer", "status"], + ).unwrap(); + + static ref INGEST_JOB_TIME: HistogramVec = HistogramVec::new( + HistogramOpts::new("ingest_job_time", "Time taken for ingest jobs"), + &["stream", "consumer"] ).unwrap(); static ref DOWNLOAD_METADATA_INSERTED_COUNT: IntCounter = IntCounter::new( @@ -61,12 +65,12 @@ lazy_static::lazy_static! { static ref INGEST_TASKS: IntGaugeVec = IntGaugeVec::new( Opts::new("ingest_tasks", "Number of tasks spawned for ingest"), - &["stream"] + &["stream", "consumer"] ).unwrap(); static ref ACK_TASKS: IntGaugeVec = IntGaugeVec::new( Opts::new("ack_tasks", "Number of tasks spawned for ack redis messages"), - &["stream"] + &["stream", "consumer"] ).unwrap(); static ref GRPC_TASKS: IntGaugeVec = IntGaugeVec::new( @@ -74,7 +78,6 @@ lazy_static::lazy_static! { &["label","stream"] ).unwrap(); - static ref BUBBLEGUM_TREE_TOTAL_LEAVES: IntGaugeVec = IntGaugeVec::new( Opts::new("bubblegum_tree_total_leaves", "Total number of leaves in the bubblegum tree"), &["tree"] @@ -118,8 +121,8 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(REDIS_XREAD_COUNT); register!(REDIS_XACK_COUNT); register!(PGPOOL_CONNECTIONS); - register!(PROGRAM_TRANSFORMER_TASKS); register!(PROGRAM_TRANSFORMER_TASK_STATUS_COUNT); + register!(INGEST_JOB_TIME); register!(DOWNLOAD_METADATA_INSERTED_COUNT); register!(INGEST_TASKS); register!(ACK_TASKS); @@ -188,6 +191,12 @@ pub fn redis_xlen_set(stream: &str, len: usize) { .set(len as i64); } +pub fn ingest_job_time_set(stream: &str, consumer: &str, value: f64) { + INGEST_JOB_TIME + .with_label_values(&[stream, consumer]) + .observe(value); +} + pub fn redis_xadd_status_inc(stream: &str, label: &str, status: Result<(), ()>, delta: usize) { REDIS_XADD_STATUS_COUNT .with_label_values(&[ @@ -198,15 +207,15 @@ pub fn redis_xadd_status_inc(stream: &str, label: &str, status: Result<(), ()>, .inc_by(delta as u64); } -pub fn redis_xread_inc(stream: &str, delta: usize) { +pub fn redis_xread_inc(stream: &str, consumer: &str, delta: usize) { REDIS_XREAD_COUNT - .with_label_values(&[stream]) + .with_label_values(&[stream, consumer]) .inc_by(delta as u64) } -pub fn redis_xack_inc(stream: &str, delta: usize) { +pub fn redis_xack_inc(stream: &str, consumer: &str, delta: usize) { REDIS_XACK_COUNT - .with_label_values(&[stream]) + .with_label_values(&[stream, consumer]) .inc_by(delta as u64) } @@ -225,20 +234,20 @@ pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { .set(size as i64) } -pub fn ingest_tasks_total_inc(stream: &str) { - INGEST_TASKS.with_label_values(&[stream]).inc() +pub fn ingest_tasks_total_inc(stream: &str, consumer: &str) { + INGEST_TASKS.with_label_values(&[stream, consumer]).inc() } -pub fn ingest_tasks_total_dec(stream: &str) { - INGEST_TASKS.with_label_values(&[stream]).dec() +pub fn ingest_tasks_total_dec(stream: &str, consumer: &str) { + INGEST_TASKS.with_label_values(&[stream, consumer]).dec() } -pub fn ack_tasks_total_inc(stream: &str) { - ACK_TASKS.with_label_values(&[stream]).inc() +pub fn ack_tasks_total_inc(stream: &str, consumer: &str) { + ACK_TASKS.with_label_values(&[stream, consumer]).inc() } -pub fn ack_tasks_total_dec(stream: &str) { - ACK_TASKS.with_label_values(&[stream]).dec() +pub fn ack_tasks_total_dec(stream: &str, consumer: &str) { + ACK_TASKS.with_label_values(&[stream, consumer]).dec() } pub fn grpc_tasks_total_inc(label: &str, stream: &str) { @@ -353,9 +362,13 @@ impl ProgramTransformerTaskStatusKind { } } -pub fn program_transformer_task_status_inc(kind: ProgramTransformerTaskStatusKind) { +pub fn program_transformer_task_status_inc( + stream: &str, + consumer: &str, + kind: ProgramTransformerTaskStatusKind, +) { PROGRAM_TRANSFORMER_TASK_STATUS_COUNT - .with_label_values(&[kind.to_str()]) + .with_label_values(&[stream, consumer, kind.to_str()]) .inc() } diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 9903bb5f4..9da08f0ab 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -2,7 +2,7 @@ use { crate::{ config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, prom::{ - ack_tasks_total_dec, ack_tasks_total_inc, ingest_tasks_total_dec, + ack_tasks_total_dec, ack_tasks_total_inc, ingest_job_time_set, ingest_tasks_total_dec, ingest_tasks_total_inc, program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, redis_xread_inc, ProgramTransformerTaskStatusKind, }, @@ -313,7 +313,7 @@ impl Acknowledge { config.name, response, count ); - redis_xack_inc(&config.name, count); + redis_xack_inc(&config.name, &config.consumer, count); } Err(e) => { error!( @@ -324,7 +324,7 @@ impl Acknowledge { } } - ack_tasks_total_dec(&config.name); + ack_tasks_total_dec(&config.name, &config.consumer); } } @@ -424,25 +424,29 @@ impl IngestStream { let ack_tx = ack_tx.clone(); let config = Arc::clone(&config); - ingest_tasks_total_inc(&config.name); + ingest_tasks_total_inc(&config.name, &config.consumer); tasks.push(tokio::spawn(async move { + let start_time = tokio::time::Instant::now(); let result = handler.handle(map).await.map_err(IngestMessageError::into); + let elapsed_time = start_time.elapsed().as_secs_f64(); + + ingest_job_time_set(&config.name, &config.consumer, elapsed_time); match result { Ok(()) => { - program_transformer_task_status_inc(ProgramTransformerTaskStatusKind::Success); + program_transformer_task_status_inc(&config.name, &config.consumer, ProgramTransformerTaskStatusKind::Success); } Err(IngestMessageError::RedisStreamMessage(e)) => { error!("Failed to process message: {:?}", e); - program_transformer_task_status_inc(e.into()); + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); } Err(IngestMessageError::DownloadMetadataJson(e)) => { - program_transformer_task_status_inc(e.into()); + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); } Err(IngestMessageError::ProgramTransformer(e)) => { error!("Failed to process message: {:?}", e); - program_transformer_task_status_inc(e.into()); + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); } } @@ -450,7 +454,7 @@ impl IngestStream { error!(target: "ingest_stream", "action=send_ack stream={} error={:?}", &config.name, e); } - ingest_tasks_total_dec(&config.name); + ingest_tasks_total_dec(&config.name, &config.consumer); })); } } @@ -487,7 +491,7 @@ impl IngestStream { let handler = Arc::clone(&handler); - ack_tasks_total_inc(&config.name); + ack_tasks_total_inc(&config.name, &config.consumer); tasks.push(tokio::spawn(async move { handler.handle(ids).await; @@ -503,7 +507,7 @@ impl IngestStream { let ids = std::mem::take(&mut pending); let handler = Arc::clone(&handler); - ack_tasks_total_inc(&config.name); + ack_tasks_total_inc(&config.name, &config.consumer); tasks.push(tokio::spawn(async move { handler.handle(ids).await; @@ -586,7 +590,7 @@ impl IngestStream { let count = ids.len(); debug!(target: "ingest_stream", "action=xread stream={} count={:?}", &config.name, count); - redis_xread_inc(&config.name, count); + redis_xread_inc(&config.name, &config.consumer, count); if let Err(e) = msg_tx.send(ids).await { error!(target: "ingest_stream", "action=send_ids stream={} error={:?}", &config.name, e); From 4baaca679d4c484375104a2042fa9ce901a6db94 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 27 Nov 2024 16:31:00 +0100 Subject: [PATCH 68/72] Use Example Configs (#180) Co-authored-by: Ahzam Akhtar --- grpc-ingest/.gitignore | 3 +++ grpc-ingest/README.md | 10 +++++++++- ...ig-grpc2redis.yml => config-grpc2redis.example.yml} | 0 ...config-ingester.yml => config-ingester.example.yml} | 0 .../{config-monitor.yml => config-monitor.example.yml} | 0 5 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 grpc-ingest/.gitignore rename grpc-ingest/{config-grpc2redis.yml => config-grpc2redis.example.yml} (100%) rename grpc-ingest/{config-ingester.yml => config-ingester.example.yml} (100%) rename grpc-ingest/{config-monitor.yml => config-monitor.example.yml} (100%) diff --git a/grpc-ingest/.gitignore b/grpc-ingest/.gitignore new file mode 100644 index 000000000..191f3badd --- /dev/null +++ b/grpc-ingest/.gitignore @@ -0,0 +1,3 @@ +config-grpc2redis.yaml +config-ingester.yaml +config-monitor.yaml diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md index 011b2a062..1f3a46393 100644 --- a/grpc-ingest/README.md +++ b/grpc-ingest/README.md @@ -14,7 +14,15 @@ docker compose up db redis prometheus INIT_FILE_PATH=./init.sql sea migrate up --database-url=postgres://solana:solana@localhost:5432/solana ``` -### Config for grpc2redis [./config-grpc2redis.yml](./config-grpc2redis.yml) +### Configs + +Example config files are available at +- [./config-grpc2redis.example.yml](./config-grpc2redis.example.yml) +- [./config-ingester.example.yml](./config-ingester.example.yml) +- [./config-monitor.example.yml](./config-monitor.example.yml) + +Copy these files and modify them as needed to setup the project. + ### Run grpc2redis service diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.example.yml similarity index 100% rename from grpc-ingest/config-grpc2redis.yml rename to grpc-ingest/config-grpc2redis.example.yml diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.example.yml similarity index 100% rename from grpc-ingest/config-ingester.yml rename to grpc-ingest/config-ingester.example.yml diff --git a/grpc-ingest/config-monitor.yml b/grpc-ingest/config-monitor.example.yml similarity index 100% rename from grpc-ingest/config-monitor.yml rename to grpc-ingest/config-monitor.example.yml From c2351257e97ec3208ed0bd408726b6fdf61cad02 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Wed, 18 Dec 2024 12:02:45 -0500 Subject: [PATCH 69/72] Respect commitment level set in grpc config (#191) --- grpc-ingest/src/grpc.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs index 480975382..ed6985dfa 100644 --- a/grpc-ingest/src/grpc.rs +++ b/grpc-ingest/src/grpc.rs @@ -140,6 +140,7 @@ impl SubscriptionTask { let request = SubscribeRequest { accounts: req_accounts, transactions: req_transactions, + commitment: Some(config.geyser.commitment.to_proto().into()), ..Default::default() }; From 9f3d543cbbc5ddd303eb8824f47d9c64000bace5 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 20 Jan 2025 09:47:33 +0100 Subject: [PATCH 70/72] Use tokio join set ingest stream task tracking --- grpc-ingest/src/redis.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 9da08f0ab..06f98c4d2 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -17,7 +17,10 @@ use { }, solana_sdk::{pubkey::Pubkey, signature::Signature}, std::{collections::HashMap, marker::PhantomData, sync::Arc}, - tokio::time::{sleep, Duration}, + tokio::{ + task::JoinSet, + time::{sleep, Duration}, + }, tracing::{debug, error, warn}, yellowstone_grpc_proto::{ convert_from::{ @@ -408,7 +411,7 @@ impl IngestStream { let config_messages = Arc::clone(&config); let messages = tokio::spawn(async move { - let mut tasks = FuturesUnordered::new(); + let mut tasks = JoinSet::new(); let config = Arc::clone(&config_messages); let handler = handler.clone(); @@ -417,7 +420,7 @@ impl IngestStream { Some(ids) = msg_rx.recv() => { for StreamId { id, map } in ids { if tasks.len() >= config.max_concurrency { - tasks.next().await; + tasks.join_next().await; } let handler = handler.clone(); @@ -426,7 +429,7 @@ impl IngestStream { ingest_tasks_total_inc(&config.name, &config.consumer); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let start_time = tokio::time::Instant::now(); let result = handler.handle(map).await.map_err(IngestMessageError::into); let elapsed_time = start_time.elapsed().as_secs_f64(); @@ -455,7 +458,7 @@ impl IngestStream { } ingest_tasks_total_dec(&config.name, &config.consumer); - })); + }); } } _ = &mut msg_shutdown_rx => { @@ -464,13 +467,13 @@ impl IngestStream { } } - while (tasks.next().await).is_some() {} + while (tasks.join_next().await).is_some() {} }); let ack = tokio::spawn({ let config = Arc::clone(&config); let mut pending = Vec::new(); - let mut tasks = FuturesUnordered::new(); + let mut tasks = JoinSet::new(); let handler = Arc::new(Acknowledge::new(Arc::clone(&config), connection.clone())); async move { @@ -484,7 +487,7 @@ impl IngestStream { if pending.len() >= config.xack_batch_max_size { if tasks.len() >= config.ack_concurrency { - tasks.next().await; + tasks.join_next().await; } let ids = std::mem::take(&mut pending); @@ -493,25 +496,25 @@ impl IngestStream { ack_tasks_total_inc(&config.name, &config.consumer); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { handler.handle(ids).await; - })); + }); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } } _ = &mut deadline, if !pending.is_empty() => { if tasks.len() >= config.ack_concurrency { - tasks.next().await; + tasks.join_next().await; } let ids = std::mem::take(&mut pending); let handler = Arc::clone(&handler); ack_tasks_total_inc(&config.name, &config.consumer); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { handler.handle(ids).await; - })); + }); deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); } @@ -526,7 +529,7 @@ impl IngestStream { handler.handle(std::mem::take(&mut pending)).await; } - while (tasks.next().await).is_some() {} + while (tasks.join_next().await).is_some() {} } }); From 3d7a88df5623a2c46db02e8cb794f9d059ac9065 Mon Sep 17 00:00:00 2001 From: Nagaprasad V R <81755170+Nagaprasadvr@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:01:59 +0530 Subject: [PATCH 71/72] Add metrics to track metadata_json_download success and failure (#207) Metadata download status metrics --------- Co-authored-by: Kyle Espinola --- Api.Dockerfile | 2 +- Builder.Dockerfile | 4 +-- Ingest.Dockerfile | 2 +- Load.Dockerfile | 2 +- Migrator.Dockerfile | 2 +- Proxy.Dockerfile | 2 +- digital_asset_types/src/dapi/change_logs.rs | 2 +- digital_asset_types/tests/common.rs | 2 ++ grpc-ingest/src/config.rs | 23 ++++----------- grpc-ingest/src/prom.rs | 21 ++++++++++---- grpc-ingest/src/redis.rs | 31 +++++++++++++++------ nft_ingester/src/backfiller.rs | 1 + program_transformers/src/bubblegum/db.rs | 2 +- rust-toolchain.toml | 2 +- 14 files changed, 56 insertions(+), 42 deletions(-) diff --git a/Api.Dockerfile b/Api.Dockerfile index cbc4ee6ee..a0b55a29f 100644 --- a/Api.Dockerfile +++ b/Api.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Builder.Dockerfile b/Builder.Dockerfile index 6ee7cbaed..e883c7c8a 100644 --- a/Builder.Dockerfile +++ b/Builder.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.75-bullseye AS builder +FROM rust:1.79-bullseye AS builder RUN apt-get update -y && \ apt-get install -y build-essential make git @@ -21,6 +21,6 @@ WORKDIR /rust RUN --mount=type=cache,target=/rust/target,id=das-rust \ cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins -FROM rust:1.75-slim-bullseye as final +FROM rust:1.79-slim-bullseye as final COPY --from=builder /rust/bins /das/ CMD echo "Built the DAS API bins!" diff --git a/Ingest.Dockerfile b/Ingest.Dockerfile index a0a6023d6..8ff559bfe 100644 --- a/Ingest.Dockerfile +++ b/Ingest.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Load.Dockerfile b/Load.Dockerfile index ae4ee3ec1..91e754c4b 100644 --- a/Load.Dockerfile +++ b/Load.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Migrator.Dockerfile b/Migrator.Dockerfile index d275aa7d6..e21f7d77e 100644 --- a/Migrator.Dockerfile +++ b/Migrator.Dockerfile @@ -1,6 +1,6 @@ FROM das-api/builder AS files -FROM rust:1.75-bullseye +FROM rust:1.79-bullseye COPY init.sql /init.sql ENV INIT_FILE_PATH=/init.sql COPY --from=files /das/migration /bins/migration diff --git a/Proxy.Dockerfile b/Proxy.Dockerfile index add959fab..cfee4fb6d 100644 --- a/Proxy.Dockerfile +++ b/Proxy.Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.75-bullseye AS builder +FROM rust:1.79-bullseye AS builder RUN cargo install wasm-pack RUN mkdir /rust diff --git a/digital_asset_types/src/dapi/change_logs.rs b/digital_asset_types/src/dapi/change_logs.rs index fdd19ddf9..4742696f0 100644 --- a/digital_asset_types/src/dapi/change_logs.rs +++ b/digital_asset_types/src/dapi/change_logs.rs @@ -206,7 +206,7 @@ fn build_asset_proof( let mut final_node_list = vec![SimpleChangeLog::default(); req_indexes.len()]; for node in required_nodes.iter() { if node.level < final_node_list.len().try_into().unwrap() { - final_node_list[node.level as usize] = node.to_owned(); + node.clone_into(&mut final_node_list[node.level as usize]) } } for (i, (n, nin)) in final_node_list diff --git a/digital_asset_types/tests/common.rs b/digital_asset_types/tests/common.rs index 6f40f610e..414cff714 100644 --- a/digital_asset_types/tests/common.rs +++ b/digital_asset_types/tests/common.rs @@ -31,8 +31,10 @@ pub struct MockMetadataArgs { /// Since we cannot easily change Metadata, we add the new DataV2 fields here at the end. pub token_standard: Option, /// Collection + #[allow(dead_code)] pub collection: Option, /// Uses + #[allow(dead_code)] pub uses: Option, pub creators: Vec, } diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index b34ba8fe8..9699215bc 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -111,19 +111,6 @@ impl ConfigIngestStream { } } -#[derive(Debug, Clone, Deserialize, Default)] -#[serde(default)] -pub struct ConfigTopograph { - #[serde(default = "ConfigTopograph::default_num_threads")] - pub num_threads: usize, -} - -impl ConfigTopograph { - pub const fn default_num_threads() -> usize { - 5 - } -} - #[derive(Debug, Clone, Default, Deserialize)] #[serde(default)] pub struct ConfigPrometheus { @@ -297,12 +284,12 @@ pub struct ConfigIngesterDownloadMetadata { default = "ConfigIngesterDownloadMetadata::default_num_threads", deserialize_with = "deserialize_usize_str" )] - pub num_threads: usize, + pub _num_threads: usize, #[serde( default = "ConfigIngesterDownloadMetadata::default_max_attempts", deserialize_with = "deserialize_usize_str" )] - pub max_attempts: usize, + pub _max_attempts: usize, #[serde( default = "ConfigIngesterDownloadMetadata::default_request_timeout", deserialize_with = "deserialize_duration_str", @@ -318,13 +305,13 @@ pub struct ConfigIngesterDownloadMetadata { default = "ConfigIngesterDownloadMetadata::default_stream_max_size", deserialize_with = "deserialize_usize_str" )] - pub pipeline_max_size: usize, + pub _pipeline_max_size: usize, #[serde( default = "ConfigIngesterDownloadMetadata::default_pipeline_max_idle", deserialize_with = "deserialize_duration_str", rename = "pipeline_max_idle_ms" )] - pub pipeline_max_idle: Duration, + pub _pipeline_max_idle: Duration, } impl ConfigIngesterDownloadMetadata { @@ -366,7 +353,7 @@ pub struct ConfigBubblegumVerify { default = "ConfigBubblegumVerify::default_report_interval", deserialize_with = "deserialize_duration_str" )] - pub report_interval: Duration, + pub _report_interval: Duration, #[serde(default)] pub only_trees: Option>, #[serde( diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs index 1bb3a125a..b4ff567fe 100644 --- a/grpc-ingest/src/prom.rs +++ b/grpc-ingest/src/prom.rs @@ -9,8 +9,7 @@ use { }, program_transformers::error::ProgramTransformerError, prometheus::{ - HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, Opts, Registry, - TextEncoder, + HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder, }, std::{net::SocketAddr, sync::Once}, tracing::{error, info}, @@ -59,8 +58,9 @@ lazy_static::lazy_static! { &["stream", "consumer"] ).unwrap(); - static ref DOWNLOAD_METADATA_INSERTED_COUNT: IntCounter = IntCounter::new( - "download_metadata_inserted_count", "Total number of inserted tasks for download metadata" + static ref DOWNLOAD_METADATA_FETCHED_COUNT: IntGaugeVec = IntGaugeVec::new( + Opts::new("download_metadata_fetched_count", "Status of download metadata task"), + &["status"] ).unwrap(); static ref INGEST_TASKS: IntGaugeVec = IntGaugeVec::new( @@ -123,7 +123,7 @@ pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { register!(PGPOOL_CONNECTIONS); register!(PROGRAM_TRANSFORMER_TASK_STATUS_COUNT); register!(INGEST_JOB_TIME); - register!(DOWNLOAD_METADATA_INSERTED_COUNT); + register!(DOWNLOAD_METADATA_FETCHED_COUNT); register!(INGEST_TASKS); register!(ACK_TASKS); register!(GRPC_TASKS); @@ -175,7 +175,10 @@ fn metrics_handler() -> Response { error!("could not encode custom metrics: {}", error); String::new() }); - Response::builder().body(Body::from(metrics)).unwrap() + Response::builder() + .header("content-type", "text/plain") + .body(Body::from(metrics)) + .unwrap() } fn not_found_handler() -> Response { @@ -258,6 +261,12 @@ pub fn grpc_tasks_total_dec(label: &str, stream: &str) { GRPC_TASKS.with_label_values(&[label, stream]).dec() } +pub fn download_metadata_json_task_status_count_inc(status: u16) { + DOWNLOAD_METADATA_FETCHED_COUNT + .with_label_values(&[&status.to_string()]) + .inc(); +} + #[derive(Debug, Clone, Copy)] pub enum ProgramTransformerTaskStatusKind { Success, diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 06f98c4d2..44c4289ba 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -2,12 +2,16 @@ use { crate::{ config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, prom::{ - ack_tasks_total_dec, ack_tasks_total_inc, ingest_job_time_set, ingest_tasks_total_dec, - ingest_tasks_total_inc, program_transformer_task_status_inc, redis_xack_inc, - redis_xlen_set, redis_xread_inc, ProgramTransformerTaskStatusKind, + ack_tasks_total_dec, ack_tasks_total_inc, download_metadata_json_task_status_count_inc, + ingest_job_time_set, ingest_tasks_total_dec, ingest_tasks_total_inc, + program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, redis_xread_inc, + ProgramTransformerTaskStatusKind, }, }, - das_core::{DownloadMetadata, DownloadMetadataInfo}, + das_core::{ + DownloadMetadata, DownloadMetadataInfo, FetchMetadataJsonError, MetadataJsonTaskError, + StatusCode, + }, futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}, program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, redis::{ @@ -202,10 +206,21 @@ impl MessageHandler for DownloadMetadataJsonHandle { Box::pin(async move { let info = DownloadMetadataInfo::try_parse_msg(input)?; - download_metadata - .handle_download(&info) - .await - .map_err(Into::into) + let response = download_metadata.handle_download(&info).await; + let status = + if let Err(MetadataJsonTaskError::Fetch(FetchMetadataJsonError::Response { + status: StatusCode::Code(code), + .. + })) = response + { + code.as_u16() + } else { + 200 + }; + + download_metadata_json_task_status_count_inc(status); + + response.map_err(IngestMessageError::DownloadMetadataJson) }) } } diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs index 7fbb1c214..39d8106c2 100644 --- a/nft_ingester/src/backfiller.rs +++ b/nft_ingester/src/backfiller.rs @@ -66,6 +66,7 @@ const BLOCK_CACHE_SIZE: usize = 300_000; const MAX_CACHE_COST: i64 = 32; const BLOCK_CACHE_DURATION: u64 = 172800; +#[allow(dead_code)] struct SlotSeq(u64, u64); /// Main public entry point for backfiller task. pub fn setup_backfiller( diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index 4c8c33f4a..470fab631 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -504,7 +504,7 @@ where pub async fn upsert_asset_creators( txn: &T, id: Vec, - creators: &Vec, + creators: &[Creator], slot_updated: i64, seq: i64, ) -> ProgramTransformerResult<()> diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eb84afdfc..402b06845 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.75.0" +channel = "1.79.0" components = ["clippy", "rustfmt"] targets = [] profile = "minimal" From 44ce53f05b3a6efe0a495d17f1f1cf6b6ff9b6a2 Mon Sep 17 00:00:00 2001 From: Nagaprasad V R <81755170+Nagaprasadvr@users.noreply.github.com> Date: Tue, 21 Jan 2025 18:27:25 +0530 Subject: [PATCH 72/72] add retry config to download_metadata_json task (#208) --- bubblegum/src/backfill/worker/tree.rs | 8 +++- bubblegum/src/lib.rs | 7 ++- core/src/metadata_json.rs | 63 +++++++++++++++++++++---- grpc-ingest/config-ingester.example.yml | 4 ++ grpc-ingest/src/config.rs | 21 ++++++++- grpc-ingest/src/ingester.rs | 16 +++++-- grpc-ingest/src/redis.rs | 22 +++++---- ops/src/account/nft.rs | 12 +++-- ops/src/account/program.rs | 12 +++-- ops/src/account/single.rs | 14 ++++-- 10 files changed, 138 insertions(+), 41 deletions(-) diff --git a/bubblegum/src/backfill/worker/tree.rs b/bubblegum/src/backfill/worker/tree.rs index e02c21305..761b674da 100644 --- a/bubblegum/src/backfill/worker/tree.rs +++ b/bubblegum/src/backfill/worker/tree.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ backfill::gap::{TreeGapFill, TreeGapModel}, tree::TreeResponse, @@ -5,7 +7,7 @@ use crate::{ }; use anyhow::Result; use clap::Parser; -use das_core::MetadataJsonDownloadWorkerArgs; +use das_core::{DownloadMetadataJsonRetryConfig, MetadataJsonDownloadWorkerArgs}; use digital_asset_types::dao::cl_audits_v2; use log::error; use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector}; @@ -44,10 +46,12 @@ impl TreeWorkerArgs { let signature_worker_args = self.signature_worker.clone(); let gap_worker_args = self.gap_worker.clone(); let force = self.force; + let download_config = Arc::new(DownloadMetadataJsonRetryConfig::default()); tokio::spawn(async move { let (metadata_json_download_worker, metadata_json_download_sender) = - metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; + metadata_json_download_worker_args + .start(metadata_json_download_db_pool, download_config)?; let (program_transformer_worker, transaction_info_sender) = program_transformer_worker_args diff --git a/bubblegum/src/lib.rs b/bubblegum/src/lib.rs index 463526337..9f800f128 100644 --- a/bubblegum/src/lib.rs +++ b/bubblegum/src/lib.rs @@ -2,6 +2,7 @@ mod backfill; mod error; mod tree; +use das_core::DownloadMetadataJsonRetryConfig; use das_core::{MetadataJsonDownloadWorkerArgs, Rpc}; pub use error::ErrorKind; mod verify; @@ -19,6 +20,7 @@ use sea_orm::{EntityTrait, QueryFilter}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Signature; use std::str::FromStr; +use std::sync::Arc; use tracing::error; use tree::TreeResponse; @@ -123,9 +125,10 @@ pub async fn start_bubblegum_replay( let metadata_json_download_db_pool = context.database_pool.clone(); let program_transformer_context = context.clone(); let signature_context = context.clone(); - + let download_config = Arc::new(DownloadMetadataJsonRetryConfig::default()); let (metadata_json_download_worker, metadata_json_download_sender) = - metadata_json_download_worker_args.start(metadata_json_download_db_pool)?; + metadata_json_download_worker_args + .start(metadata_json_download_db_pool, download_config)?; let (program_transformer_worker, transaction_info_sender) = program_transformer_worker_args .start(program_transformer_context, metadata_json_download_sender)?; diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs index 36c3ebf0a..e0fd9c521 100644 --- a/core/src/metadata_json.rs +++ b/core/src/metadata_json.rs @@ -8,6 +8,7 @@ use { reqwest::{Client, Url as ReqwestUrl}, sea_orm::{entity::*, SqlxPostgresConnector}, serde::{Deserialize, Serialize}, + std::{sync::Arc, time::Duration}, tokio::{ sync::mpsc::{error::SendError, unbounded_channel, UnboundedSender}, task::JoinHandle, @@ -22,6 +23,33 @@ pub struct DownloadMetadataInfo { slot: i64, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadMetadataJsonRetryConfig { + pub retries: u8, + pub retry_max_delay_ms: u64, + pub retry_min_delay_ms: u64, +} + +impl Default for DownloadMetadataJsonRetryConfig { + fn default() -> Self { + Self { + retries: 1, + retry_max_delay_ms: 10, + retry_min_delay_ms: 1, + } + } +} + +impl DownloadMetadataJsonRetryConfig { + pub const fn new(retries: usize, retry_max_delay_ms: usize, retry_min_delay_ms: usize) -> Self { + Self { + retries: retries as u8, + retry_max_delay_ms: retry_max_delay_ms as u64, + retry_min_delay_ms: retry_min_delay_ms as u64, + } + } +} + impl DownloadMetadataInfo { pub fn new(asset_data_id: Vec, uri: String, slot: i64) -> Self { Self { @@ -69,6 +97,7 @@ impl MetadataJsonDownloadWorkerArgs { pub fn start( &self, pool: sqlx::PgPool, + config: Arc, ) -> Result< (JoinHandle<()>, UnboundedSender), MetadataJsonDownloadWorkerError, @@ -80,9 +109,9 @@ impl MetadataJsonDownloadWorkerArgs { self.metadata_json_download_worker_request_timeout, )) .build()?; - let handle = tokio::spawn(async move { let mut handlers = FuturesUnordered::new(); + let download_config = Arc::clone(&config); while let Some(download_metadata_info) = rx.recv().await { if handlers.len() >= worker_count { @@ -92,7 +121,12 @@ impl MetadataJsonDownloadWorkerArgs { let pool = pool.clone(); let client = client.clone(); - handlers.push(spawn_task(client, pool, download_metadata_info)); + handlers.push(spawn_task( + client, + pool, + download_metadata_info, + Arc::clone(&download_config), + )); } while handlers.next().await.is_some() {} @@ -116,13 +150,16 @@ fn spawn_task( client: Client, pool: sqlx::PgPool, download_metadata_info: DownloadMetadataInfo, + config: Arc, ) -> JoinHandle<()> { tokio::spawn(async move { let timing = Instant::now(); let asset_data_id = bs58::encode(download_metadata_info.asset_data_id.clone()).into_string(); - if let Err(e) = perform_metadata_json_task(client, pool, &download_metadata_info).await { + if let Err(e) = + perform_metadata_json_task(client, pool, &download_metadata_info, config).await + { error!("Asset {} failed: {}", asset_data_id, e); } @@ -162,6 +199,7 @@ pub enum StatusCode { async fn fetch_metadata_json( client: Client, metadata_json_url: &str, + config: Arc, ) -> Result { (|| async { let url = ReqwestUrl::parse(metadata_json_url)?; @@ -187,7 +225,12 @@ async fn fetch_metadata_json( } } }) - .retry(&ExponentialBuilder::default()) + .retry( + &ExponentialBuilder::default() + .with_max_times(config.retries.into()) + .with_min_delay(Duration::from_millis(config.retry_min_delay_ms)) + .with_max_delay(Duration::from_millis(config.retry_max_delay_ms)), + ) .await } @@ -205,8 +248,9 @@ pub async fn perform_metadata_json_task( client: Client, pool: sqlx::PgPool, download_metadata_info: &DownloadMetadataInfo, -) -> Result { - match fetch_metadata_json(client, &download_metadata_info.uri).await { + config: Arc, +) -> Result<(), MetadataJsonTaskError> { + match fetch_metadata_json(client, &download_metadata_info.uri, config).await { Ok(metadata) => { let active_model = asset_data::ActiveModel { id: Set(download_metadata_info.asset_data_id.clone()), @@ -217,9 +261,9 @@ pub async fn perform_metadata_json_task( let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - let model = active_model.update(&conn).await?; + active_model.update(&conn).await?; - Ok(model) + Ok(()) } Err(e) => Err(MetadataJsonTaskError::Fetch(e)), } @@ -238,13 +282,14 @@ impl DownloadMetadata { pub async fn handle_download( &self, download_metadata_info: &DownloadMetadataInfo, + config: Arc, ) -> Result<(), MetadataJsonTaskError> { perform_metadata_json_task( self.client.clone(), self.pool.clone(), download_metadata_info, + config, ) .await - .map(|_| ()) } } diff --git a/grpc-ingest/config-ingester.example.yml b/grpc-ingest/config-ingester.example.yml index 37815047e..e9dbda4eb 100644 --- a/grpc-ingest/config-ingester.example.yml +++ b/grpc-ingest/config-ingester.example.yml @@ -51,5 +51,9 @@ transactions: download_metadata: # maximum number of attempts for downloading metadata JSON. max_attempts: 1 + # reretry max delay in milliseconds + retry_max_delay: 10 + # retry min delay in milliseconds + retry_min_delay: 1 stream: name: METADATA_JSON diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs index 9699215bc..14acfd32e 100644 --- a/grpc-ingest/src/config.rs +++ b/grpc-ingest/src/config.rs @@ -289,7 +289,7 @@ pub struct ConfigIngesterDownloadMetadata { default = "ConfigIngesterDownloadMetadata::default_max_attempts", deserialize_with = "deserialize_usize_str" )] - pub _max_attempts: usize, + pub max_attempts: usize, #[serde( default = "ConfigIngesterDownloadMetadata::default_request_timeout", deserialize_with = "deserialize_duration_str", @@ -312,6 +312,17 @@ pub struct ConfigIngesterDownloadMetadata { rename = "pipeline_max_idle_ms" )] pub _pipeline_max_idle: Duration, + + #[serde( + default = "ConfigIngesterDownloadMetadata::default_retry_max_delay_ms", + deserialize_with = "deserialize_usize_str" + )] + pub retry_max_delay_ms: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_retry_min_delay_ms", + deserialize_with = "deserialize_usize_str" + )] + pub retry_min_delay_ms: usize, } impl ConfigIngesterDownloadMetadata { @@ -338,6 +349,14 @@ impl ConfigIngesterDownloadMetadata { pub const fn default_request_timeout() -> Duration { Duration::from_millis(3_000) } + + pub const fn default_retry_max_delay_ms() -> usize { + 10 + } + + pub const fn default_retry_min_delay_ms() -> usize { + 1 + } } #[derive(Debug, Clone, Deserialize)] diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index 710415d5f..c72c456d0 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -6,7 +6,10 @@ use { redis::{AccountHandle, DownloadMetadataJsonHandle, IngestStream, TransactionHandle}, util::create_shutdown, }, - das_core::{DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier}, + das_core::{ + DownloadMetadata, DownloadMetadataInfo, DownloadMetadataJsonRetryConfig, + DownloadMetadataNotifier, + }, futures::{future::BoxFuture, stream::StreamExt}, program_transformers::ProgramTransformer, redis::aio::MultiplexedConnection, @@ -80,9 +83,14 @@ pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { let download_metadatas = IngestStream::build() .config(config.download_metadata.stream.clone()) .connection(connection.clone()) - .handler(DownloadMetadataJsonHandle::new(Arc::clone( - &download_metadata, - ))) + .handler(DownloadMetadataJsonHandle::new( + Arc::clone(&download_metadata), + Arc::new(DownloadMetadataJsonRetryConfig::new( + config.download_metadata.max_attempts, + config.download_metadata.retry_max_delay_ms, + config.download_metadata.retry_min_delay_ms, + )), + )) .start() .await?; diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs index 44c4289ba..7e240a817 100644 --- a/grpc-ingest/src/redis.rs +++ b/grpc-ingest/src/redis.rs @@ -9,10 +9,10 @@ use { }, }, das_core::{ - DownloadMetadata, DownloadMetadataInfo, FetchMetadataJsonError, MetadataJsonTaskError, - StatusCode, + DownloadMetadata, DownloadMetadataInfo, DownloadMetadataJsonRetryConfig, + FetchMetadataJsonError, MetadataJsonTaskError, StatusCode, }, - futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}, + futures::future::BoxFuture, program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, redis::{ aio::MultiplexedConnection, @@ -195,7 +195,7 @@ pub trait MessageHandler: Send + Sync + Clone + 'static { ) -> BoxFuture<'static, Result<(), IngestMessageError>>; } -pub struct DownloadMetadataJsonHandle(Arc); +pub struct DownloadMetadataJsonHandle(Arc, Arc); impl MessageHandler for DownloadMetadataJsonHandle { fn handle( @@ -203,10 +203,13 @@ impl MessageHandler for DownloadMetadataJsonHandle { input: HashMap, ) -> BoxFuture<'static, Result<(), IngestMessageError>> { let download_metadata = Arc::clone(&self.0); + let download_config = Arc::clone(&self.1); Box::pin(async move { let info = DownloadMetadataInfo::try_parse_msg(input)?; - let response = download_metadata.handle_download(&info).await; + let response = download_metadata + .handle_download(&info, download_config) + .await; let status = if let Err(MetadataJsonTaskError::Fetch(FetchMetadataJsonError::Response { status: StatusCode::Code(code), @@ -226,14 +229,17 @@ impl MessageHandler for DownloadMetadataJsonHandle { } impl DownloadMetadataJsonHandle { - pub fn new(download_metadata: Arc) -> Self { - Self(download_metadata) + pub fn new( + download_metadata: Arc, + config: Arc, + ) -> Self { + Self(download_metadata, config) } } impl Clone for DownloadMetadataJsonHandle { fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) + Self(Arc::clone(&self.0), Arc::clone(&self.1)) } } diff --git a/ops/src/account/nft.rs b/ops/src/account/nft.rs index f92e005d0..1f3d2ad20 100644 --- a/ops/src/account/nft.rs +++ b/ops/src/account/nft.rs @@ -8,8 +8,8 @@ use log::error; use clap::Parser; use das_core::{ - connect_db, create_download_metadata_notifier, MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, - SolanaRpcArgs, + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, }; use mpl_token_metadata::accounts::Metadata; use program_transformers::ProgramTransformer; @@ -42,9 +42,11 @@ pub async fn run(config: Args) -> Result<()> { let pool = connect_db(&config.database).await?; let metadata_json_download_db_pool = pool.clone(); - let (metadata_json_download_worker, metadata_json_download_sender) = config - .metadata_json_download_worker - .start(metadata_json_download_db_pool)?; + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; let download_metadata_notifier = create_download_metadata_notifier(metadata_json_download_sender.clone()).await; diff --git a/ops/src/account/program.rs b/ops/src/account/program.rs index e25431f52..590ef140d 100644 --- a/ops/src/account/program.rs +++ b/ops/src/account/program.rs @@ -2,8 +2,8 @@ use super::account_info; use anyhow::Result; use clap::Parser; use das_core::{ - connect_db, create_download_metadata_notifier, MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, - SolanaRpcArgs, + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, }; use futures::{stream::FuturesUnordered, StreamExt}; use log::error; @@ -55,9 +55,11 @@ pub async fn run(config: Args) -> Result<()> { let metadata_json_download_db_pool = pool.clone(); - let (metadata_json_download_worker, metadata_json_download_sender) = config - .metadata_json_download_worker - .start(metadata_json_download_db_pool)?; + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; let (tx, mut rx) = mpsc::channel::>(config.max_buffer_size); let download_metadata_notifier = diff --git a/ops/src/account/single.rs b/ops/src/account/single.rs index 81d6e0ad4..d62aa3ed6 100644 --- a/ops/src/account/single.rs +++ b/ops/src/account/single.rs @@ -1,10 +1,12 @@ +use std::sync::Arc; + use anyhow::Result; use super::account_info; use clap::Parser; use das_core::{ - connect_db, create_download_metadata_notifier, MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, - SolanaRpcArgs, + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, }; use program_transformers::ProgramTransformer; use solana_sdk::pubkey::Pubkey; @@ -35,9 +37,11 @@ pub async fn run(config: Args) -> Result<()> { let pool = connect_db(&config.database).await?; let metadata_json_download_db_pool = pool.clone(); - let (metadata_json_download_worker, metadata_json_download_sender) = config - .metadata_json_download_worker - .start(metadata_json_download_db_pool)?; + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; { let download_metadata_notifier =