From dd8103a2fdbf877d62dd21824ead7af834c01fe3 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 12 Aug 2024 14:18:53 +0200 Subject: [PATCH] fix: special use of das-metadata-json in triton-build --- Cargo.lock | 35 ++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ core/src/metadata_json.rs | 2 +- grpc-ingest/src/ingester.rs | 9 ++------ metadata_json/src/cmds/backfill.rs | 4 ++-- metadata_json/src/cmds/ingest.rs | 4 ++-- metadata_json/src/cmds/report.rs | 4 ++-- metadata_json/src/cmds/single.rs | 4 ++-- nft_ingester/Cargo.toml | 1 + 9 files changed, 49 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd49b2476..93c05dfda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1875,6 +1875,40 @@ dependencies = [ "yellowstone-grpc-tools", ] +[[package]] +name = "das-metadata-json" +version = "0.7.2" +dependencies = [ + "anyhow", + "backon", + "bs58 0.4.0", + "cadence", + "cadence-macros", + "chrono", + "clap 4.4.8", + "das-core", + "derive_more", + "digital_asset_types", + "env_logger 0.10.0", + "figment", + "futures", + "indicatif", + "log", + "plerkle_messenger", + "rand 0.8.5", + "redis 0.25.4", + "reqwest", + "sea-orm", + "sea-query 0.28.5", + "serde", + "serde_json", + "sqlx", + "thiserror", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "das-ops" version = "0.7.2" @@ -3794,6 +3828,7 @@ dependencies = [ "chrono", "clap 4.4.8", "das-core", + "das-metadata-json", "digital_asset_types", "figment", "flatbuffers", diff --git a/Cargo.toml b/Cargo.toml index 8346aa30f..28bc32cfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "digital_asset_types", "integration_tests", "metaplex-rpc-proxy", + "metadata_json", "migration", "nft_ingester", "ops", @@ -48,6 +49,7 @@ cargo-lock = "9.0.0" chrono = "0.4.19" clap = "4.2.2" das-core = { path = "core" } +das-metadata-json = { path = "metadata_json" } das-backfill = { path = "backfill" } das_api = { path = "das_api" } derive_more = { version = "0.99.17" } diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs index eec14a46e..25174ae3c 100644 --- a/core/src/metadata_json.rs +++ b/core/src/metadata_json.rs @@ -53,7 +53,7 @@ pub async fn create_download_metadata_notifier( }) } -#[derive(Parser, Clone, Debug)] +#[derive(Parser, Clone, Debug, PartialEq, Eq)] pub struct MetadataJsonDownloadWorkerArgs { /// The number of worker threads #[arg(long, env, default_value = "25")] diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs index bf92e8c85..a96ee2de0 100644 --- a/grpc-ingest/src/ingester.rs +++ b/grpc-ingest/src/ingester.rs @@ -1,7 +1,7 @@ use { crate::{ config::{ConfigIngester, ConfigIngesterDownloadMetadata}, - download_metadata::{TASK_TYPE}, + download_metadata::TASK_TYPE, postgres::{create_pool as pg_create_pool, metrics_pgpool}, prom::{ download_metadata_inserted_total_inc, program_transformer_task_status_inc, @@ -16,10 +16,7 @@ use { }, chrono::Utc, crypto::{digest::Digest, sha2::Sha256}, - das_core::{ - DownloadMetadata, DownloadMetadataInfo, - DownloadMetadataNotifier, - }, + das_core::{DownloadMetadata, DownloadMetadataInfo, DownloadMetadataNotifier}, digital_asset_types::dao::{sea_orm_active_enums::TaskStatus, tasks}, futures::{ future::{pending, BoxFuture, FusedFuture, FutureExt}, @@ -203,8 +200,6 @@ pub async fn run_v2(config: ConfigIngester) -> anyhow::Result<()> { } else { debug!("Message acknowledged successfully"); } - - } } } diff --git a/metadata_json/src/cmds/backfill.rs b/metadata_json/src/cmds/backfill.rs index 28070c05a..bbbaf4b90 100644 --- a/metadata_json/src/cmds/backfill.rs +++ b/metadata_json/src/cmds/backfill.rs @@ -30,9 +30,9 @@ pub struct BackfillArgs { pub async fn run(args: BackfillArgs) -> Result<(), anyhow::Error> { let batch_size = args.batch_size; - let pool = connect_db(args.database).await?; + let pool = connect_db(&args.database).await?; - setup_metrics(args.metrics)?; + setup_metrics(&args.metrics)?; let client = ClientBuilder::new() .timeout(Duration::from_millis(args.timeout)) diff --git a/metadata_json/src/cmds/ingest.rs b/metadata_json/src/cmds/ingest.rs index d87f70bab..df801572b 100644 --- a/metadata_json/src/cmds/ingest.rs +++ b/metadata_json/src/cmds/ingest.rs @@ -27,9 +27,9 @@ pub struct IngestArgs { pub async fn run(args: IngestArgs) -> Result<(), anyhow::Error> { let rx = Receiver::try_from_config(args.receiver.into()).await?; - let pool = connect_db(args.database).await?; + let pool = connect_db(&args.database).await?; - setup_metrics(args.metrics)?; + setup_metrics(&args.metrics)?; let client = ClientBuilder::new() .timeout(Duration::from_millis(args.timeout)) diff --git a/metadata_json/src/cmds/report.rs b/metadata_json/src/cmds/report.rs index 4daecb94b..d37bedc58 100644 --- a/metadata_json/src/cmds/report.rs +++ b/metadata_json/src/cmds/report.rs @@ -21,9 +21,9 @@ pub struct ReportArgs { } pub async fn run(args: ReportArgs) -> Result<(), anyhow::Error> { - let pool = connect_db(args.database).await?; + let pool = connect_db(&args.database).await?; - setup_metrics(args.metrics)?; + setup_metrics(&args.metrics)?; loop { { diff --git a/metadata_json/src/cmds/single.rs b/metadata_json/src/cmds/single.rs index 5510b9d8f..c50a8b0c8 100644 --- a/metadata_json/src/cmds/single.rs +++ b/metadata_json/src/cmds/single.rs @@ -20,9 +20,9 @@ pub struct SingleArgs { } pub async fn run(args: SingleArgs) -> Result<(), anyhow::Error> { - let pool = connect_db(args.database).await?; + let pool = connect_db(&args.database).await?; - setup_metrics(args.metrics)?; + setup_metrics(&args.metrics)?; let asset_data = bs58::decode(args.mint.as_str()).into_vec()?; diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index 8d6185025..cb6e1cf01 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -15,6 +15,7 @@ cadence-macros = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive", "cargo"] } das-core = { workspace = true } +das-metadata-json = { workspace = true } digital_asset_types = { workspace = true, features = [ "json_types", "sql_types",