From bcb68b64a409b23701dd6542ad039c0b75e22c41 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 19:52:16 +0000 Subject: [PATCH 1/8] checkpoint --- staking/Cargo.lock | 1 + staking/cli/Cargo.toml | 1 + staking/cli/src/cli.rs | 1 + staking/cli/src/instructions.rs | 410 +++++++++++++++++- staking/cli/src/main.rs | 4 + staking/programs/staking/src/lib.rs | 2 +- .../staking/src/utils/voter_weight.rs | 2 +- 7 files changed, 416 insertions(+), 5 deletions(-) diff --git a/staking/Cargo.lock b/staking/Cargo.lock index 7e1c4523..7aaf4f9d 100644 --- a/staking/Cargo.lock +++ b/staking/Cargo.lock @@ -5289,6 +5289,7 @@ dependencies = [ "serde_json", "serde_wormhole", "shellexpand", + "solana-account-decoder", "solana-client", "solana-remote-wallet", "solana-sdk", diff --git a/staking/cli/Cargo.toml b/staking/cli/Cargo.toml index 47302437..3b562f35 100644 --- a/staking/cli/Cargo.toml +++ b/staking/cli/Cargo.toml @@ -25,3 +25,4 @@ reqwest = "0.11" serde_json = "1.0.128" uriparse = "0.6.4" solana-remote-wallet = "1.18.16" +solana-account-decoder = "1.18.16" diff --git a/staking/cli/src/cli.rs b/staking/cli/src/cli.rs index 10fdb75a..1c1a8108 100644 --- a/staking/cli/src/cli.rs +++ b/staking/cli/src/cli.rs @@ -109,6 +109,7 @@ pub enum Action { #[clap(long, help = "Publisher caps")] publisher_caps: Pubkey, }, + GetAllAccounts {}, } pub enum SignerSource { diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index 277ab620..df0303b1 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -1,14 +1,22 @@ use { anchor_lang::{ + accounts::account, AccountDeserialize, + Discriminator, InstructionData, ToAccountMetas, }, anchor_spl::{ associated_token::spl_associated_token_account, - token::spl_token, + token::{ + spl_token, + TokenAccount, + }, + }, + base64::{ + encode, + Engine, }, - base64::Engine, integration_tests::{ integrity_pool::pda::{ get_delegation_record_address, @@ -42,9 +50,19 @@ use { }, reqwest::blocking::Client, serde_wormhole::RawMessage, + solana_account_decoder::UiAccountEncoding, solana_client::{ rpc_client::RpcClient, - rpc_config::RpcSendTransactionConfig, + rpc_config::{ + RpcAccountInfoConfig, + RpcProgramAccountsConfig, + RpcSendTransactionConfig, + }, + rpc_filter::{ + Memcmp, + MemcmpEncodedBytes, + RpcFilterType, + }, }, solana_sdk::{ commitment_config::CommitmentConfig, @@ -67,10 +85,31 @@ use { TransactionError, }, }, + staking::{ + state::{ + global_config::GlobalConfig, + max_voter_weight_record::MAX_VOTER_WEIGHT, + positions::{ + DynamicPositionArray, + DynamicPositionArrayAccount, + PositionData, + PositionState, + }, + stake_account::StakeAccountMetadataV2, + vesting::VestingSchedule, + }, + utils::voter_weight::compute_voter_weight, + }, std::{ cmp::min, + collections::{ + HashMap, + HashSet, + }, convert::TryInto, mem::size_of, + str::FromStr, + thread::current, }, wormhole_core_bridge_solana::sdk::{ WriteEncodedVaaArgs, @@ -498,6 +537,12 @@ pub fn initialize_pool( .unwrap(); } +pub fn get_current_time(rpc_client: &RpcClient) -> i64 { + let slot = rpc_client.get_slot().unwrap(); + let blocktime = rpc_client.get_block_time(slot).unwrap(); + blocktime +} + pub fn get_current_epoch(rpc_client: &RpcClient) -> u64 { let slot = rpc_client.get_slot().unwrap(); let blocktime = rpc_client.get_block_time(slot).unwrap(); @@ -825,3 +870,362 @@ pub fn update_y(rpc_client: &RpcClient, signer: &dyn Signer, y: u64) { process_transaction(rpc_client, &[instruction], &[signer]).unwrap(); } + +pub fn get_all_accounts(rpc_client: &RpcClient) { + let data: Vec<(Pubkey, DynamicPositionArrayAccount, Pubkey, Pubkey)> = rpc_client + .get_program_accounts_with_config( + &staking::ID, + RpcProgramAccountsConfig { + filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( + 0, + MemcmpEncodedBytes::Bytes(PositionData::discriminator().to_vec()), + ))]), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64Zstd), + data_slice: None, + commitment: None, + min_context_slot: None, + }, + with_context: None, + }, + ) + .unwrap() + .into_iter() + .map(|(pubkey, account)| { + ( + pubkey, + DynamicPositionArrayAccount { + key: pubkey, + lamports: account.lamports, + data: account.data.clone(), + }, + get_stake_account_metadata_address(pubkey), + get_stake_account_custody_address(pubkey), + ) + }) + .collect::>(); + + let metadata_accounts_data = rpc_client + .get_program_accounts_with_config( + &staking::ID, + RpcProgramAccountsConfig { + filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( + 0, + MemcmpEncodedBytes::Bytes(StakeAccountMetadataV2::discriminator().to_vec()), + ))]), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64Zstd), + data_slice: None, + commitment: None, + min_context_slot: None, + }, + with_context: None, + }, + ) + .unwrap() + .into_iter() + .map(|(pubkey, account)| { + ( + pubkey, + StakeAccountMetadataV2::try_deserialize(&mut account.data.as_slice()).unwrap(), + ) + }) + .collect::>(); + + let config = GlobalConfig::try_deserialize( + &mut rpc_client + .get_account_data(&get_config_address()) + .unwrap() + .as_slice(), + ) + .unwrap(); + let current_time = get_current_time(rpc_client); + + let locked_metadata_account_data: HashMap = metadata_accounts_data + .iter() + .map(|(pubkey, metadata)| { + ( + *pubkey, + metadata + .lock + .get_unvested_balance(current_time, config.pyth_token_list_time) + .unwrap(), + ) + }) + .collect::>(); + + let data = data + .into_iter() + .map(|(pubkey, account, metadata_pubkey, custody_pubkey)| { + ( + pubkey, + account, + metadata_pubkey, + custody_pubkey, + *locked_metadata_account_data.get(&custody_pubkey).unwrap(), + ) + }) + .collect::>(); + + + let locked_token_accounts_pubkeys = data + .iter() + .filter(|(_, _, _, _, locked_amount)| *locked_amount > 0u64) + .map(|(_, _, _, token_account_pubkey, _)| *token_account_pubkey) + .collect::>(); + + let mut actual_amounts: HashMap = HashMap::new(); + for chunk in locked_token_accounts_pubkeys.chunks(100) { + rpc_client + .get_multiple_accounts(chunk) + .unwrap() + .into_iter() + .enumerate() + .for_each(|(index, account)| { + actual_amounts.insert( + chunk[index], + TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()) + .unwrap() + .amount, + ); + }); + } + + let data = data + .into_iter() + .map( + |(pubkey, account, metadata_pubkey, custody_pubkey, locked_amount)| { + ( + pubkey, + account, + metadata_pubkey, + custody_pubkey, + min( + locked_amount, + *actual_amounts.get(&custody_pubkey).unwrap_or(&0u64), + ), + ) + }, + ) + .collect::>(); + + let current_epoch = get_current_epoch(rpc_client); + let data = data + .into_iter() + .map( + |(pubkey, mut account, metadata_pubkey, custody_pubkey, locked_amount)| { + let dynamic_position_array = account.to_dynamic_position_array(); + let owner = dynamic_position_array.owner().unwrap(); + let staked_in_governance = compute_voter_weight( + &dynamic_position_array, + current_epoch, + MAX_VOTER_WEIGHT, + MAX_VOTER_WEIGHT, + ) + .unwrap(); + + let staked_in_ois = { + let mut amount = 0u64; + for i in 0..dynamic_position_array.get_position_capacity() { + if let Some(position) = dynamic_position_array.read_position(i).unwrap() { + match position.get_current_position(current_epoch).unwrap() { + PositionState::LOCKED | PositionState::PREUNLOCKING => { + if !position.is_voting() { + amount += position.amount; + } + } + _ => {} + } + } + } + amount + }; + ( + pubkey, + metadata_pubkey, + custody_pubkey, + owner, + locked_amount, + staked_in_governance, + staked_in_ois, + ) + }, + ) + .collect::>(); + + use std::{ + fs::File, + io::{ + BufWriter, + Write, + }, + }; + + let file = File::create("output.csv").unwrap(); + let mut writer = BufWriter::new(file); + + // Write the header + writeln!(writer, "pubkey,account,metadata_pubkey,custody_pubkey,owner,locked_amount,staked_in_governance,staked_in_ois").unwrap(); + // Write the data + for ( + pubkey, + metadata_pubkey, + custody_pubkey, + owner, + locked_amount, + staked_in_governance, + staked_in_ois, + ) in data + { + writeln!( + writer, + "{},{},{},{},{},{},{}", + pubkey, + metadata_pubkey, + custody_pubkey, + owner, + locked_amount, + staked_in_governance, + staked_in_ois + ) + .unwrap(); + } + + + // let current_epoch = get_current_epoch(rpc_client); + + // let governance_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, + // account))| { let mut account = DynamicPositionArrayAccount{ + // key: *pubkey, + // lamports: account.lamports, + // data: account.data.clone(), + // }; + // let dynamic_position_array = account.to_dynamic_position_array(); + + // println!("Account {}", index); + + + // compute_voter_weight( + // &dynamic_position_array, + // current_epoch, + // MAX_VOTER_WEIGHT, + // MAX_VOTER_WEIGHT + // ).unwrap() + // }).collect::>(); + + // let ois_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, account))| + // { let mut account = DynamicPositionArrayAccount{ + // key: *pubkey, + // lamports: account.lamports, + // data: account.data.clone(), + // }; + // let dynamic_position_array = account.to_dynamic_position_array(); + + // let mut raw_voter_weight = 0u64; + // for i in 0..dynamic_position_array.get_position_capacity() { + // if let Some(position) = dynamic_position_array.read_position(i).unwrap(){ + // match position.get_current_position(current_epoch).unwrap() { + // PositionState::LOCKED | PositionState::PREUNLOCKING => { + // if !position.is_voting() { + // // position.amount is trusted, so I don't think this can overflow, + // // but still probably better to use checked math + // raw_voter_weight = raw_voter_weight + position.amount; + // } + // } + // _ => {} + // } + // } + // } + // raw_voter_weight + // }).collect::>(); + + // let total_capacity: u64 = governance_stake_amounts.iter().sum(); + // println!("Total capacity: {}", total_capacity); + // println!("Total OIS capacity: {}", ois_stake_amounts.iter().sum::()); + + + // let mut frequencies = std::collections::HashMap::new(); + // for capacity in &position_account_capacities { + // *frequencies.entry(*capacity).or_insert(0) += 1; + // } + + // let mut sorted_frequencies: Vec<_> = frequencies.iter().collect(); + // sorted_frequencies.sort_by_key(|&(capacity, _)| capacity); + // for (capacity, count) in sorted_frequencies { + // println!("{}: {}", capacity, count); + // } + + + let stake_account_metadata_accounts = rpc_client + .get_program_accounts_with_config( + &staking::ID, + RpcProgramAccountsConfig { + filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( + 0, + MemcmpEncodedBytes::Bytes(StakeAccountMetadataV2::discriminator().to_vec()), + ))]), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64Zstd), + data_slice: None, + commitment: None, + min_context_slot: None, + }, + with_context: None, + }, + ) + .unwrap() + .into_iter() + .map(|(pubkey, account)| { + ( + pubkey, + StakeAccountMetadataV2::try_deserialize(&mut account.data.as_slice()).unwrap(), + ) + }) + .collect::>(); + + let stake_account_metadata_accounts = stake_account_metadata_accounts + .iter() + .filter(|(_, metadata)| !matches!(metadata.lock, VestingSchedule::FullyVested)) + .collect::>(); + println!("{}", stake_account_metadata_accounts.len()); + + let mut custody_accounts: Vec = stake_account_metadata_accounts + .iter() + .map(|(pubkey, _)| get_stake_account_custody_address(*pubkey)) + .collect::>(); + + let current_time = get_current_time(rpc_client); + let stake_account_locked_amount = stake_account_metadata_accounts + .iter() + .map(|(pubkey, metadata)| { + metadata + .lock + .get_unvested_balance(current_time, config.pyth_token_list_time) + .unwrap() + }) + .collect::>(); + + println!("{:?}", stake_account_locked_amount.iter().sum::()); + + let mut token_accounts: Vec = Vec::new(); + for chunk in custody_accounts.chunks(100) { + let batch = rpc_client + .get_multiple_accounts(&chunk) + .unwrap() + .into_iter() + .map(|account| { + TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()).unwrap() + }) + .collect::>(); + token_accounts.extend_from_slice(batch.as_slice()); + } + + + let custody_accounts = rpc_client.get_token_largest_accounts( + &Pubkey::from_str("HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3").unwrap(), + ); + + // println!("{:?}", positio); + // println!("{:?}", stake_account_metadata_accounts.unwrap().len()); + println!("{:?}", custody_accounts.unwrap().len()); +} diff --git a/staking/cli/src/main.rs b/staking/cli/src/main.rs index 5c6a0cc8..c42d783c 100644 --- a/staking/cli/src/main.rs +++ b/staking/cli/src/main.rs @@ -11,6 +11,7 @@ use { close_publisher_caps, create_slash_event, fetch_publisher_caps_and_advance, + get_all_accounts, initialize_pool, initialize_reward_custody, set_publisher_stake_account, @@ -93,5 +94,8 @@ fn main() { Action::ClosePublisherCaps { publisher_caps } => { close_publisher_caps(&rpc_client, keypair.as_ref(), publisher_caps) } + Action::GetAllAccounts {} => { + get_all_accounts(&rpc_client); + } } } diff --git a/staking/programs/staking/src/lib.rs b/staking/programs/staking/src/lib.rs index fc3226a4..c2412cea 100644 --- a/staking/programs/staking/src/lib.rs +++ b/staking/programs/staking/src/lib.rs @@ -41,7 +41,7 @@ use { pub mod context; pub mod error; pub mod state; -mod utils; +pub mod utils; #[cfg(feature = "wasm")] pub mod wasm; diff --git a/staking/programs/staking/src/utils/voter_weight.rs b/staking/programs/staking/src/utils/voter_weight.rs index 5b14920b..1695514d 100644 --- a/staking/programs/staking/src/utils/voter_weight.rs +++ b/staking/programs/staking/src/utils/voter_weight.rs @@ -24,7 +24,7 @@ pub fn compute_voter_weight( if position.is_voting() { // position.amount is trusted, so I don't think this can overflow, // but still probably better to use checked math - raw_voter_weight = raw_voter_weight.checked_add(position.amount).unwrap(); + raw_voter_weight = raw_voter_weight + position.amount; } } _ => {} From f4293c0b488c2572f2a6ec965edd0c4a2aa40957 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 20:10:35 +0000 Subject: [PATCH 2/8] checkpoint --- staking/cli/src/instructions.rs | 267 ++++++++++++++++---------------- 1 file changed, 131 insertions(+), 136 deletions(-) diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index df0303b1..a3883fb0 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -962,7 +962,7 @@ pub fn get_all_accounts(rpc_client: &RpcClient) { account, metadata_pubkey, custody_pubkey, - *locked_metadata_account_data.get(&custody_pubkey).unwrap(), + *locked_metadata_account_data.get(&metadata_pubkey).unwrap(), ) }) .collect::>(); @@ -1065,7 +1065,7 @@ pub fn get_all_accounts(rpc_client: &RpcClient) { let mut writer = BufWriter::new(file); // Write the header - writeln!(writer, "pubkey,account,metadata_pubkey,custody_pubkey,owner,locked_amount,staked_in_governance,staked_in_ois").unwrap(); + writeln!(writer, "pubkey,metadata_pubkey,custody_pubkey,owner,locked_amount,staked_in_governance,staked_in_ois").unwrap(); // Write the data for ( pubkey, @@ -1091,141 +1091,136 @@ pub fn get_all_accounts(rpc_client: &RpcClient) { .unwrap(); } - - // let current_epoch = get_current_epoch(rpc_client); - - // let governance_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, - // account))| { let mut account = DynamicPositionArrayAccount{ - // key: *pubkey, - // lamports: account.lamports, - // data: account.data.clone(), - // }; - // let dynamic_position_array = account.to_dynamic_position_array(); - - // println!("Account {}", index); - - - // compute_voter_weight( - // &dynamic_position_array, - // current_epoch, - // MAX_VOTER_WEIGHT, - // MAX_VOTER_WEIGHT - // ).unwrap() - // }).collect::>(); - - // let ois_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, account))| - // { let mut account = DynamicPositionArrayAccount{ - // key: *pubkey, - // lamports: account.lamports, - // data: account.data.clone(), - // }; - // let dynamic_position_array = account.to_dynamic_position_array(); - - // let mut raw_voter_weight = 0u64; - // for i in 0..dynamic_position_array.get_position_capacity() { - // if let Some(position) = dynamic_position_array.read_position(i).unwrap(){ - // match position.get_current_position(current_epoch).unwrap() { - // PositionState::LOCKED | PositionState::PREUNLOCKING => { - // if !position.is_voting() { - // // position.amount is trusted, so I don't think this can overflow, - // // but still probably better to use checked math - // raw_voter_weight = raw_voter_weight + position.amount; - // } - // } - // _ => {} - // } - // } - // } - // raw_voter_weight - // }).collect::>(); - - // let total_capacity: u64 = governance_stake_amounts.iter().sum(); - // println!("Total capacity: {}", total_capacity); - // println!("Total OIS capacity: {}", ois_stake_amounts.iter().sum::()); - - - // let mut frequencies = std::collections::HashMap::new(); - // for capacity in &position_account_capacities { - // *frequencies.entry(*capacity).or_insert(0) += 1; - // } - - // let mut sorted_frequencies: Vec<_> = frequencies.iter().collect(); - // sorted_frequencies.sort_by_key(|&(capacity, _)| capacity); - // for (capacity, count) in sorted_frequencies { - // println!("{}: {}", capacity, count); + // // let current_epoch = get_current_epoch(rpc_client); + + // // let governance_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, + // // account))| { let mut account = DynamicPositionArrayAccount{ + // // key: *pubkey, + // // lamports: account.lamports, + // // data: account.data.clone(), + // // }; + // // let dynamic_position_array = account.to_dynamic_position_array(); + + // // println!("Account {}", index); + + // // compute_voter_weight( + // // &dynamic_position_array, + // // current_epoch, + // // MAX_VOTER_WEIGHT, + // // MAX_VOTER_WEIGHT + // // ).unwrap() + // // }).collect::>(); + + // // let ois_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, + // account))| // { let mut account = DynamicPositionArrayAccount{ + // // key: *pubkey, + // // lamports: account.lamports, + // // data: account.data.clone(), + // // }; + // // let dynamic_position_array = account.to_dynamic_position_array(); + + // // let mut raw_voter_weight = 0u64; + // // for i in 0..dynamic_position_array.get_position_capacity() { + // // if let Some(position) = dynamic_position_array.read_position(i).unwrap(){ + // // match position.get_current_position(current_epoch).unwrap() { + // // PositionState::LOCKED | PositionState::PREUNLOCKING => { + // // if !position.is_voting() { + // // // position.amount is trusted, so I don't think this can overflow, + // // // but still probably better to use checked math + // // raw_voter_weight = raw_voter_weight + position.amount; + // // } + // // } + // // _ => {} + // // } + // // } + // // } + // // raw_voter_weight + // // }).collect::>(); + + // // let total_capacity: u64 = governance_stake_amounts.iter().sum(); + // // println!("Total capacity: {}", total_capacity); + // // println!("Total OIS capacity: {}", ois_stake_amounts.iter().sum::()); + + // // let mut frequencies = std::collections::HashMap::new(); + // // for capacity in &position_account_capacities { + // // *frequencies.entry(*capacity).or_insert(0) += 1; + // // } + + // // let mut sorted_frequencies: Vec<_> = frequencies.iter().collect(); + // // sorted_frequencies.sort_by_key(|&(capacity, _)| capacity); + // // for (capacity, count) in sorted_frequencies { + // // println!("{}: {}", capacity, count); + // // } + + // let stake_account_metadata_accounts = rpc_client + // .get_program_accounts_with_config( + // &staking::ID, + // RpcProgramAccountsConfig { + // filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( + // 0, + // MemcmpEncodedBytes::Bytes(StakeAccountMetadataV2::discriminator().to_vec()), + // ))]), + // account_config: RpcAccountInfoConfig { + // encoding: Some(UiAccountEncoding::Base64Zstd), + // data_slice: None, + // commitment: None, + // min_context_slot: None, + // }, + // with_context: None, + // }, + // ) + // .unwrap() + // .into_iter() + // .map(|(pubkey, account)| { + // ( + // pubkey, + // StakeAccountMetadataV2::try_deserialize(&mut account.data.as_slice()).unwrap(), + // ) + // }) + // .collect::>(); + + // let stake_account_metadata_accounts = stake_account_metadata_accounts + // .iter() + // .filter(|(_, metadata)| !matches!(metadata.lock, VestingSchedule::FullyVested)) + // .collect::>(); + // println!("{}", stake_account_metadata_accounts.len()); + + // let mut custody_accounts: Vec = stake_account_metadata_accounts + // .iter() + // .map(|(pubkey, _)| get_stake_account_custody_address(*pubkey)) + // .collect::>(); + + // let current_time = get_current_time(rpc_client); + // let stake_account_locked_amount = stake_account_metadata_accounts + // .iter() + // .map(|(pubkey, metadata)| { + // metadata + // .lock + // .get_unvested_balance(current_time, config.pyth_token_list_time) + // .unwrap() + // }) + // .collect::>(); + + // println!("{:?}", stake_account_locked_amount.iter().sum::()); + + // let mut token_accounts: Vec = Vec::new(); + // for chunk in custody_accounts.chunks(100) { + // let batch = rpc_client + // .get_multiple_accounts(&chunk) + // .unwrap() + // .into_iter() + // .map(|account| { + // TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()).unwrap() + // }) + // .collect::>(); + // token_accounts.extend_from_slice(batch.as_slice()); // } + // let custody_accounts = rpc_client.get_token_largest_accounts( + // &Pubkey::from_str("HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3").unwrap(), + // ); - let stake_account_metadata_accounts = rpc_client - .get_program_accounts_with_config( - &staking::ID, - RpcProgramAccountsConfig { - filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( - 0, - MemcmpEncodedBytes::Bytes(StakeAccountMetadataV2::discriminator().to_vec()), - ))]), - account_config: RpcAccountInfoConfig { - encoding: Some(UiAccountEncoding::Base64Zstd), - data_slice: None, - commitment: None, - min_context_slot: None, - }, - with_context: None, - }, - ) - .unwrap() - .into_iter() - .map(|(pubkey, account)| { - ( - pubkey, - StakeAccountMetadataV2::try_deserialize(&mut account.data.as_slice()).unwrap(), - ) - }) - .collect::>(); - - let stake_account_metadata_accounts = stake_account_metadata_accounts - .iter() - .filter(|(_, metadata)| !matches!(metadata.lock, VestingSchedule::FullyVested)) - .collect::>(); - println!("{}", stake_account_metadata_accounts.len()); - - let mut custody_accounts: Vec = stake_account_metadata_accounts - .iter() - .map(|(pubkey, _)| get_stake_account_custody_address(*pubkey)) - .collect::>(); - - let current_time = get_current_time(rpc_client); - let stake_account_locked_amount = stake_account_metadata_accounts - .iter() - .map(|(pubkey, metadata)| { - metadata - .lock - .get_unvested_balance(current_time, config.pyth_token_list_time) - .unwrap() - }) - .collect::>(); - - println!("{:?}", stake_account_locked_amount.iter().sum::()); - - let mut token_accounts: Vec = Vec::new(); - for chunk in custody_accounts.chunks(100) { - let batch = rpc_client - .get_multiple_accounts(&chunk) - .unwrap() - .into_iter() - .map(|account| { - TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()).unwrap() - }) - .collect::>(); - token_accounts.extend_from_slice(batch.as_slice()); - } - - - let custody_accounts = rpc_client.get_token_largest_accounts( - &Pubkey::from_str("HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3").unwrap(), - ); - - // println!("{:?}", positio); - // println!("{:?}", stake_account_metadata_accounts.unwrap().len()); - println!("{:?}", custody_accounts.unwrap().len()); + // // println!("{:?}", positio); + // // println!("{:?}", stake_account_metadata_accounts.unwrap().len()); + // println!("{:?}", custody_accounts.unwrap().len()); } From 123f65d5e6f0f9f176c689132e0176de3fd8ed4e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 21:05:18 +0000 Subject: [PATCH 3/8] checkpoint --- staking/Cargo.lock | 1 + staking/cli/Cargo.toml | 1 + staking/cli/src/cli.rs | 2 +- staking/cli/src/instructions.rs | 140 ++------------------------------ staking/cli/src/main.rs | 6 +- 5 files changed, 11 insertions(+), 139 deletions(-) diff --git a/staking/Cargo.lock b/staking/Cargo.lock index 7aaf4f9d..97cef11a 100644 --- a/staking/Cargo.lock +++ b/staking/Cargo.lock @@ -5279,6 +5279,7 @@ dependencies = [ "anchor-spl", "base64 0.22.1", "byteorder", + "chrono", "clap 3.2.25", "integration-tests", "integrity-pool", diff --git a/staking/cli/Cargo.toml b/staking/cli/Cargo.toml index 3b562f35..9a3e3cf8 100644 --- a/staking/cli/Cargo.toml +++ b/staking/cli/Cargo.toml @@ -26,3 +26,4 @@ serde_json = "1.0.128" uriparse = "0.6.4" solana-remote-wallet = "1.18.16" solana-account-decoder = "1.18.16" +chrono = "0.4.38" diff --git a/staking/cli/src/cli.rs b/staking/cli/src/cli.rs index 1c1a8108..174572bd 100644 --- a/staking/cli/src/cli.rs +++ b/staking/cli/src/cli.rs @@ -109,7 +109,7 @@ pub enum Action { #[clap(long, help = "Publisher caps")] publisher_caps: Pubkey, }, - GetAllAccounts {}, + SaveStakeAccountsSnapshot {}, } pub enum SignerSource { diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index a3883fb0..8504401d 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -107,9 +107,11 @@ use { HashSet, }, convert::TryInto, + fmt::Debug, mem::size_of, str::FromStr, thread::current, + time::SystemTime, }, wormhole_core_bridge_solana::sdk::{ WriteEncodedVaaArgs, @@ -871,7 +873,7 @@ pub fn update_y(rpc_client: &RpcClient, signer: &dyn Signer, y: u64) { process_transaction(rpc_client, &[instruction], &[signer]).unwrap(); } -pub fn get_all_accounts(rpc_client: &RpcClient) { +pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { let data: Vec<(Pubkey, DynamicPositionArrayAccount, Pubkey, Pubkey)> = rpc_client .get_program_accounts_with_config( &staking::ID, @@ -1061,7 +1063,8 @@ pub fn get_all_accounts(rpc_client: &RpcClient) { }, }; - let file = File::create("output.csv").unwrap(); + let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H:%M:%S").to_string(); + let file = File::create(format!("snapshots/snapshot-{}.csv", timestamp)).unwrap(); let mut writer = BufWriter::new(file); // Write the header @@ -1090,137 +1093,4 @@ pub fn get_all_accounts(rpc_client: &RpcClient) { ) .unwrap(); } - - // // let current_epoch = get_current_epoch(rpc_client); - - // // let governance_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, - // // account))| { let mut account = DynamicPositionArrayAccount{ - // // key: *pubkey, - // // lamports: account.lamports, - // // data: account.data.clone(), - // // }; - // // let dynamic_position_array = account.to_dynamic_position_array(); - - // // println!("Account {}", index); - - // // compute_voter_weight( - // // &dynamic_position_array, - // // current_epoch, - // // MAX_VOTER_WEIGHT, - // // MAX_VOTER_WEIGHT - // // ).unwrap() - // // }).collect::>(); - - // // let ois_stake_amounts = position_accounts.iter().enumerate().map(|(index, (pubkey, - // account))| // { let mut account = DynamicPositionArrayAccount{ - // // key: *pubkey, - // // lamports: account.lamports, - // // data: account.data.clone(), - // // }; - // // let dynamic_position_array = account.to_dynamic_position_array(); - - // // let mut raw_voter_weight = 0u64; - // // for i in 0..dynamic_position_array.get_position_capacity() { - // // if let Some(position) = dynamic_position_array.read_position(i).unwrap(){ - // // match position.get_current_position(current_epoch).unwrap() { - // // PositionState::LOCKED | PositionState::PREUNLOCKING => { - // // if !position.is_voting() { - // // // position.amount is trusted, so I don't think this can overflow, - // // // but still probably better to use checked math - // // raw_voter_weight = raw_voter_weight + position.amount; - // // } - // // } - // // _ => {} - // // } - // // } - // // } - // // raw_voter_weight - // // }).collect::>(); - - // // let total_capacity: u64 = governance_stake_amounts.iter().sum(); - // // println!("Total capacity: {}", total_capacity); - // // println!("Total OIS capacity: {}", ois_stake_amounts.iter().sum::()); - - // // let mut frequencies = std::collections::HashMap::new(); - // // for capacity in &position_account_capacities { - // // *frequencies.entry(*capacity).or_insert(0) += 1; - // // } - - // // let mut sorted_frequencies: Vec<_> = frequencies.iter().collect(); - // // sorted_frequencies.sort_by_key(|&(capacity, _)| capacity); - // // for (capacity, count) in sorted_frequencies { - // // println!("{}: {}", capacity, count); - // // } - - // let stake_account_metadata_accounts = rpc_client - // .get_program_accounts_with_config( - // &staking::ID, - // RpcProgramAccountsConfig { - // filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new( - // 0, - // MemcmpEncodedBytes::Bytes(StakeAccountMetadataV2::discriminator().to_vec()), - // ))]), - // account_config: RpcAccountInfoConfig { - // encoding: Some(UiAccountEncoding::Base64Zstd), - // data_slice: None, - // commitment: None, - // min_context_slot: None, - // }, - // with_context: None, - // }, - // ) - // .unwrap() - // .into_iter() - // .map(|(pubkey, account)| { - // ( - // pubkey, - // StakeAccountMetadataV2::try_deserialize(&mut account.data.as_slice()).unwrap(), - // ) - // }) - // .collect::>(); - - // let stake_account_metadata_accounts = stake_account_metadata_accounts - // .iter() - // .filter(|(_, metadata)| !matches!(metadata.lock, VestingSchedule::FullyVested)) - // .collect::>(); - // println!("{}", stake_account_metadata_accounts.len()); - - // let mut custody_accounts: Vec = stake_account_metadata_accounts - // .iter() - // .map(|(pubkey, _)| get_stake_account_custody_address(*pubkey)) - // .collect::>(); - - // let current_time = get_current_time(rpc_client); - // let stake_account_locked_amount = stake_account_metadata_accounts - // .iter() - // .map(|(pubkey, metadata)| { - // metadata - // .lock - // .get_unvested_balance(current_time, config.pyth_token_list_time) - // .unwrap() - // }) - // .collect::>(); - - // println!("{:?}", stake_account_locked_amount.iter().sum::()); - - // let mut token_accounts: Vec = Vec::new(); - // for chunk in custody_accounts.chunks(100) { - // let batch = rpc_client - // .get_multiple_accounts(&chunk) - // .unwrap() - // .into_iter() - // .map(|account| { - // TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()).unwrap() - // }) - // .collect::>(); - // token_accounts.extend_from_slice(batch.as_slice()); - // } - - // let custody_accounts = rpc_client.get_token_largest_accounts( - // &Pubkey::from_str("HZ1JovNiVvGrGNiiYvEozEVgZ58xaU3RKwX8eACQBCt3").unwrap(), - // ); - - // // println!("{:?}", positio); - // // println!("{:?}", stake_account_metadata_accounts.unwrap().len()); - // println!("{:?}", custody_accounts.unwrap().len()); } diff --git a/staking/cli/src/main.rs b/staking/cli/src/main.rs index c42d783c..a06c0b9f 100644 --- a/staking/cli/src/main.rs +++ b/staking/cli/src/main.rs @@ -11,9 +11,9 @@ use { close_publisher_caps, create_slash_event, fetch_publisher_caps_and_advance, - get_all_accounts, initialize_pool, initialize_reward_custody, + save_stake_accounts_snapshot, set_publisher_stake_account, slash, update_delegation_fee, @@ -94,8 +94,8 @@ fn main() { Action::ClosePublisherCaps { publisher_caps } => { close_publisher_caps(&rpc_client, keypair.as_ref(), publisher_caps) } - Action::GetAllAccounts {} => { - get_all_accounts(&rpc_client); + Action::SaveStakeAccountsSnapshot {} => { + save_stake_accounts_snapshot(&rpc_client); } } } From 2ab97767b7ccdad4c972d44a0bf8c021b1e51276 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 21:13:45 +0000 Subject: [PATCH 4/8] go --- staking/programs/staking/src/utils/voter_weight.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking/programs/staking/src/utils/voter_weight.rs b/staking/programs/staking/src/utils/voter_weight.rs index 1695514d..5b14920b 100644 --- a/staking/programs/staking/src/utils/voter_weight.rs +++ b/staking/programs/staking/src/utils/voter_weight.rs @@ -24,7 +24,7 @@ pub fn compute_voter_weight( if position.is_voting() { // position.amount is trusted, so I don't think this can overflow, // but still probably better to use checked math - raw_voter_weight = raw_voter_weight + position.amount; + raw_voter_weight = raw_voter_weight.checked_add(position.amount).unwrap(); } } _ => {} From 42a66df5280641919898ff0a8fc71714743fc189 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 21:16:23 +0000 Subject: [PATCH 5/8] imports --- staking/cli/src/instructions.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index 8504401d..8fcde795 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -108,6 +108,11 @@ use { }, convert::TryInto, fmt::Debug, + fs::File, + io::{ + BufWriter, + Write, + }, mem::size_of, str::FromStr, thread::current, @@ -1055,14 +1060,6 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { ) .collect::>(); - use std::{ - fs::File, - io::{ - BufWriter, - Write, - }, - }; - let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H:%M:%S").to_string(); let file = File::create(format!("snapshots/snapshot-{}.csv", timestamp)).unwrap(); let mut writer = BufWriter::new(file); From b27489c364af41b984e77562c903292af773bd54 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 21:21:58 +0000 Subject: [PATCH 6/8] clippy --- staking/cli/src/instructions.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index 8fcde795..b7b49296 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -1,6 +1,5 @@ use { anchor_lang::{ - accounts::account, AccountDeserialize, Discriminator, InstructionData, @@ -13,10 +12,7 @@ use { TokenAccount, }, }, - base64::{ - encode, - Engine, - }, + base64::Engine, integration_tests::{ integrity_pool::pda::{ get_delegation_record_address, @@ -90,33 +86,24 @@ use { global_config::GlobalConfig, max_voter_weight_record::MAX_VOTER_WEIGHT, positions::{ - DynamicPositionArray, DynamicPositionArrayAccount, PositionData, PositionState, }, stake_account::StakeAccountMetadataV2, - vesting::VestingSchedule, }, utils::voter_weight::compute_voter_weight, }, std::{ cmp::min, - collections::{ - HashMap, - HashSet, - }, + collections::HashMap, convert::TryInto, - fmt::Debug, fs::File, io::{ BufWriter, Write, }, mem::size_of, - str::FromStr, - thread::current, - time::SystemTime, }, wormhole_core_bridge_solana::sdk::{ WriteEncodedVaaArgs, @@ -546,8 +533,8 @@ pub fn initialize_pool( pub fn get_current_time(rpc_client: &RpcClient) -> i64 { let slot = rpc_client.get_slot().unwrap(); - let blocktime = rpc_client.get_block_time(slot).unwrap(); - blocktime + + rpc_client.get_block_time(slot).unwrap() } pub fn get_current_epoch(rpc_client: &RpcClient) -> u64 { From 733dd5a2eb27b41950f523573a3dfb5c56a8ec39 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 11 Nov 2024 21:26:40 +0000 Subject: [PATCH 7/8] some clean up --- staking/cli/src/instructions.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index b7b49296..49b4ffd6 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -533,7 +533,6 @@ pub fn initialize_pool( pub fn get_current_time(rpc_client: &RpcClient) -> i64 { let slot = rpc_client.get_slot().unwrap(); - rpc_client.get_block_time(slot).unwrap() } @@ -935,7 +934,7 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { .unwrap(); let current_time = get_current_time(rpc_client); - let locked_metadata_account_data: HashMap = metadata_accounts_data + let metadata_account_data_locked: HashMap = metadata_accounts_data .iter() .map(|(pubkey, metadata)| { ( @@ -956,19 +955,20 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { account, metadata_pubkey, custody_pubkey, - *locked_metadata_account_data.get(&metadata_pubkey).unwrap(), + *metadata_account_data_locked.get(&metadata_pubkey).unwrap(), ) }) .collect::>(); - + // We need to check the actual tokens accounts, since you can initialize a stake account with an + // arbitrary vesting schedule but 0 tokens let locked_token_accounts_pubkeys = data .iter() .filter(|(_, _, _, _, locked_amount)| *locked_amount > 0u64) .map(|(_, _, _, token_account_pubkey, _)| *token_account_pubkey) .collect::>(); - let mut actual_amounts: HashMap = HashMap::new(); + let mut locked_token_accounts_actual_amounts: HashMap = HashMap::new(); for chunk in locked_token_accounts_pubkeys.chunks(100) { rpc_client .get_multiple_accounts(chunk) @@ -976,7 +976,7 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { .into_iter() .enumerate() .for_each(|(index, account)| { - actual_amounts.insert( + locked_token_accounts_actual_amounts.insert( chunk[index], TokenAccount::try_deserialize(&mut account.unwrap().data.as_slice()) .unwrap() @@ -996,7 +996,9 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { custody_pubkey, min( locked_amount, - *actual_amounts.get(&custody_pubkey).unwrap_or(&0u64), + *locked_token_accounts_actual_amounts + .get(&custody_pubkey) + .unwrap_or(&0u64), ), ) }, From 30b946e2278e429859a625a06861c047de508cb0 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 12 Nov 2024 12:07:44 +0000 Subject: [PATCH 8/8] add custody authority --- staking/cli/src/instructions.rs | 54 +++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/staking/cli/src/instructions.rs b/staking/cli/src/instructions.rs index 49b4ffd6..83a575e7 100644 --- a/staking/cli/src/instructions.rs +++ b/staking/cli/src/instructions.rs @@ -865,7 +865,7 @@ pub fn update_y(rpc_client: &RpcClient, signer: &dyn Signer, y: u64) { } pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { - let data: Vec<(Pubkey, DynamicPositionArrayAccount, Pubkey, Pubkey)> = rpc_client + let data: Vec<(Pubkey, DynamicPositionArrayAccount, Pubkey, Pubkey, Pubkey)> = rpc_client .get_program_accounts_with_config( &staking::ID, RpcProgramAccountsConfig { @@ -894,6 +894,7 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { }, get_stake_account_metadata_address(pubkey), get_stake_account_custody_address(pubkey), + get_stake_account_custody_authority_address(pubkey), ) }) .collect::>(); @@ -949,23 +950,26 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { let data = data .into_iter() - .map(|(pubkey, account, metadata_pubkey, custody_pubkey)| { - ( - pubkey, - account, - metadata_pubkey, - custody_pubkey, - *metadata_account_data_locked.get(&metadata_pubkey).unwrap(), - ) - }) + .map( + |(pubkey, account, metadata_pubkey, custody_pubkey, custody_authority_pubkey)| { + ( + pubkey, + account, + metadata_pubkey, + custody_pubkey, + custody_authority_pubkey, + *metadata_account_data_locked.get(&metadata_pubkey).unwrap(), + ) + }, + ) .collect::>(); // We need to check the actual tokens accounts, since you can initialize a stake account with an // arbitrary vesting schedule but 0 tokens let locked_token_accounts_pubkeys = data .iter() - .filter(|(_, _, _, _, locked_amount)| *locked_amount > 0u64) - .map(|(_, _, _, token_account_pubkey, _)| *token_account_pubkey) + .filter(|(_, _, _, _, _, locked_amount)| *locked_amount > 0u64) + .map(|(_, _, _, token_account_pubkey, _, _)| *token_account_pubkey) .collect::>(); let mut locked_token_accounts_actual_amounts: HashMap = HashMap::new(); @@ -988,12 +992,20 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { let data = data .into_iter() .map( - |(pubkey, account, metadata_pubkey, custody_pubkey, locked_amount)| { + |( + pubkey, + account, + metadata_pubkey, + custody_pubkey, + custody_authority_pubkey, + locked_amount, + )| { ( pubkey, account, metadata_pubkey, custody_pubkey, + custody_authority_pubkey, min( locked_amount, *locked_token_accounts_actual_amounts @@ -1009,7 +1021,14 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { let data = data .into_iter() .map( - |(pubkey, mut account, metadata_pubkey, custody_pubkey, locked_amount)| { + |( + pubkey, + mut account, + metadata_pubkey, + custody_pubkey, + custody_authority_pubkey, + locked_amount, + )| { let dynamic_position_array = account.to_dynamic_position_array(); let owner = dynamic_position_array.owner().unwrap(); let staked_in_governance = compute_voter_weight( @@ -1040,6 +1059,7 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { pubkey, metadata_pubkey, custody_pubkey, + custody_authority_pubkey, owner, locked_amount, staked_in_governance, @@ -1054,12 +1074,13 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { let mut writer = BufWriter::new(file); // Write the header - writeln!(writer, "pubkey,metadata_pubkey,custody_pubkey,owner,locked_amount,staked_in_governance,staked_in_ois").unwrap(); + writeln!(writer, "positions_pubkey,metadata_pubkey,custody_pubkey,custody_authority_pubkey,owner,locked_amount,staked_in_governance,staked_in_ois").unwrap(); // Write the data for ( pubkey, metadata_pubkey, custody_pubkey, + custody_authority_pubkey, owner, locked_amount, staked_in_governance, @@ -1068,10 +1089,11 @@ pub fn save_stake_accounts_snapshot(rpc_client: &RpcClient) { { writeln!( writer, - "{},{},{},{},{},{},{}", + "{},{},{},{},{},{},{},{}", pubkey, metadata_pubkey, custody_pubkey, + custody_authority_pubkey, owner, locked_amount, staked_in_governance,