feat(backfiller): generate table and model for query last transaction…

… record for fast forwarding tree transaction crawling

digital_asset_types/Cargo.toml

@@ -7,10 +7,17 @@ publish = false
 
 [dependencies]
 spl-concurrent-merkle-tree = "0.2.0"
-sea-orm = { optional = true, version = "0.10.6", features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] }
+sea-orm = { optional = true, version = "0.10.6", features = [
+  "macros",
+  "runtime-tokio-rustls",
+  "sqlx-postgres",
+  "with-chrono",
+] }
 sea-query = { version = "0.28.1", features = ["postgres-array"] }
 serde = { version = "1.0.137", optional = true }
-serde_json = { version = "1.0.81", optional = true, features=["preserve_order"] }
+serde_json = { version = "1.0.81", optional = true, features = [
+  "preserve_order",
+] }
 bs58 = "0.4.0"
 borsh = { version = "~0.10.3", optional = true }
 borsh-derive = { version = "~0.10.3", optional = true }

digital_asset_types/src/dao/generated/mod.rs

@@ -16,3 +16,4 @@ pub mod sea_orm_active_enums;
 pub mod tasks;
 pub mod token_accounts;
 pub mod tokens;
+pub mod tree_transactions;

digital_asset_types/src/dao/generated/prelude.rs

@@ -13,3 +13,4 @@ pub use super::raw_txn::Entity as RawTxn;
 pub use super::tasks::Entity as Tasks;
 pub use super::token_accounts::Entity as TokenAccounts;
 pub use super::tokens::Entity as Tokens;
+pub use super::tree_transactions::Entity as TreeTransactions;

digital_asset_types/src/dao/generated/tree_transactions.rs

@@ -0,0 +1,67 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.5
+
+use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
+pub struct Entity;
+
+impl EntityName for Entity {
+    fn table_name(&self) -> &str {
+        "tree_transactions"
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
+pub struct Model {
+    pub signature: String,
+    pub tree: Vec<u8>,
+    pub slot: i64,
+    pub created_at: Option<DateTimeWithTimeZone>,
+    pub processed_at: Option<DateTimeWithTimeZone>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
+pub enum Column {
+    Signature,
+    Tree,
+    Slot,
+    CreatedAt,
+    ProcessedAt,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
+pub enum PrimaryKey {
+    Signature,
+}
+
+impl PrimaryKeyTrait for PrimaryKey {
+    type ValueType = String;
+    fn auto_increment() -> bool {
+        false
+    }
+}
+
+#[derive(Copy, Clone, Debug, EnumIter)]
+pub enum Relation {}
+
+impl ColumnTrait for Column {
+    type EntityName = Entity;
+    fn def(&self) -> ColumnDef {
+        match self {
+            Self::Signature => ColumnType::Char(Some(64u32)).def(),
+            Self::Tree => ColumnType::Binary.def(),
+            Self::Slot => ColumnType::BigInteger.def(),
+            Self::CreatedAt => ColumnType::TimestampWithTimeZone.def().null(),
+            Self::ProcessedAt => ColumnType::TimestampWithTimeZone.def().null(),
+        }
+    }
+}
+
+impl RelationTrait for Relation {
+    fn def(&self) -> RelationDef {
+        panic!("No RelationDef")
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

migration/src/lib.rs

@@ -32,6 +32,7 @@ mod m20230918_182123_add_raw_name_symbol;
 mod m20230919_072154_cl_audits;
 mod m20231101_120101_add_instruction_into_cl_audit;
 mod m20231101_120101_cl_audit_table_index;
+mod m20231208_103949_create_tree_transactions_table;
 
 pub struct Migrator;
 
@@ -71,6 +72,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230919_072154_cl_audits::Migration),
             Box::new(m20231101_120101_add_instruction_into_cl_audit::Migration),
             Box::new(m20231101_120101_cl_audit_table_index::Migration),
+            Box::new(m20231208_103949_create_tree_transactions_table::Migration),
         ]
     }
 }

migration/src/m20231208_103949_create_tree_transactions_table.rs

@@ -0,0 +1,61 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_table(
+                Table::create()
+                    .table(TreeTransactions::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(TreeTransactions::Signature)
+                            .char_len(64)
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(TreeTransactions::Tree).binary().not_null())
+                    .col(ColumnDef::new(TreeTransactions::Slot).big_integer().not_null())
+                    .col(ColumnDef::new(TreeTransactions::CreatedAt).timestamp_with_time_zone().default("now()"))
+                    .col(ColumnDef::new(TreeTransactions::ProcessedAt).timestamp_with_time_zone())
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_index(
+                Index::create()
+                    .name("tree_slot_index")
+                    .table(TreeTransactions::Table)
+                    .col(TreeTransactions::Tree)
+                    .col(TreeTransactions::Slot)
+                    .unique()
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_index(Index::drop().name("tree_slot_index").table(TreeTransactions::Table).to_owned())
+            .await?;
+
+        manager
+            .drop_table(Table::drop().table(TreeTransactions::Table).to_owned())
+            .await
+    }
+}
+
+/// Learn more at https://docs.rs/sea-query#iden
+#[derive(Iden)]
+enum TreeTransactions {
+    Table,
+    Signature,
+    Tree,
+    CreatedAt,
+    ProcessedAt,
+    Slot,
+}

nft_ingester/Cargo.toml

@@ -33,7 +33,6 @@ sea-orm = { version = "0.10.6", features = [
   "runtime-tokio-rustls",
   "sqlx-postgres",
   "with-chrono",
-  "mock",
 ] }
 sea-query = { version = "0.28.1", features = ["postgres-array"] }
 chrono = "0.4.19"

tools/bgtask_creator/Cargo.toml

@@ -7,15 +7,30 @@ publish = false
 [dependencies]
 anyhow = "1.0.70"
 clap = { version = "4.1.4", features = ["derive", "cargo"] }
-digital_asset_types = { path = "../../digital_asset_types", features = ["json_types", "sql_types"] }
+digital_asset_types = { path = "../../digital_asset_types", features = [
+  "json_types",
+  "sql_types",
+] }
 futures = "0.3.25"
 lazy_static = "1.4.0"
 log = "0.4.17"
 nft_ingester = { path = "../../nft_ingester" }
 prometheus = "0.13.3"
-sea-orm = { version = "0.10.6", features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] }
+sea-orm = { version = "0.10.6", features = [
+  "macros",
+  "runtime-tokio-rustls",
+  "sqlx-postgres",
+  "with-chrono",
+] }
 sea-query = { version = "0.28.1", features = ["postgres-array"] }
 solana-sdk = "~1.16.16"
-sqlx = { version = "0.6.2", features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] }
+sqlx = { version = "0.6.2", features = [
+  "macros",
+  "runtime-tokio-rustls",
+  "postgres",
+  "uuid",
+  "offline",
+  "json",
+] }
 tokio = { version = "1.23.0", features = ["macros", "rt-multi-thread"] }
 txn_forwarder = { path = "../txn_forwarder" }

tools/tree-status/Cargo.toml

@@ -11,22 +11,43 @@ anchor-client = "0.28.0"
 anyhow = "1.0.70"
 bs58 = "0.4.0"
 clap = { version = "4.1.4", features = ["derive"] }
-digital_asset_types = { path = "../../digital_asset_types", features = ["json_types", "sql_types"] }
+digital_asset_types = { path = "../../digital_asset_types", features = [
+  "json_types",
+  "sql_types",
+] }
 env_logger = "0.10.0"
 flatbuffers = "23.1.21"
 futures = "0.3.28"
 hex = "0.4.3"
 lazy_static = "1.4.0"
 log = "0.4.17"
 prometheus = "0.13.3"
-sea-orm = { version = "0.10.6", features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] }
+sea-orm = { version = "0.10.6", features = [
+  "macros",
+  "runtime-tokio-rustls",
+  "sqlx-postgres",
+  "with-chrono",
+] }
 serde_json = "1.0.81"
 solana-client = "~1.16.16"
 solana-sdk = "~1.16.16"
 solana-transaction-status = "~1.16.16"
 spl-account-compression = { version = "0.2.0", features = ["no-entrypoint"] }
 spl-noop = { version = "0.2.0", features = ["no-entrypoint"] }
-sqlx = { version = "0.6.2", features = ["macros", "runtime-tokio-rustls", "postgres", "uuid", "offline", "json"] }
+sqlx = { version = "0.6.2", features = [
+  "macros",
+  "runtime-tokio-rustls",
+  "postgres",
+  "uuid",
+  "offline",
+  "json",
+] }
 thiserror = "1.0.31"
-tokio = { version = "1.23.0", features = ["fs", "macros", "rt-multi-thread", "sync", "time"] }
+tokio = { version = "1.23.0", features = [
+  "fs",
+  "macros",
+  "rt-multi-thread",
+  "sync",
+  "time",
+] }
 txn_forwarder = { path = "../txn_forwarder" }

tree_backfiller/Cargo.toml

@@ -36,7 +36,6 @@ sea-orm = { version = "0.10.6", features = [
   "runtime-tokio-rustls",
   "sqlx-postgres",
   "with-chrono",
-  "mock",
 ] }
 sea-query = { version = "0.28.1", features = ["postgres-array"] }
 chrono = "0.4.19"