diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index 15684355a..f8e7fd198 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -12,10 +12,10 @@ on: workflow_dispatch: push: branches: - - 'main' + - "main" env: - CARGO_TERM_COLOR: always + CARGO_TERM_COLOR: always jobs: build-api: @@ -33,15 +33,15 @@ jobs: - name: set build cache uses: actions/cache@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - digital-asset-rpc-infrastructure/target/ - key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }} - restore-keys: | - ${{ matrix.os }}_digital-asset-rpc-infrastructure + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + digital-asset-rpc-infrastructure/target/ + key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }} + restore-keys: | + ${{ matrix.os }}_digital-asset-rpc-infrastructure - name: build digital asset rpc infra run: cargo build --verbose --release @@ -54,7 +54,7 @@ jobs: mv target/release/migration target/release/migration22 mv target/release/das_api target/release/das_api22 - # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them + # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them # Omiting this will save on storage consumption on the account - name: Publish artifact if: matrix.os == 'ubuntu-22.04' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ced2cd37..60f1b69ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,11 +8,11 @@ on: pull_request: push: branches: - - 'main' + - "main" workflow_dispatch: env: - CARGO_TERM_COLOR: always + CARGO_TERM_COLOR: always jobs: test: @@ -25,20 +25,20 @@ jobs: - name: set build cache uses: actions/cache@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - digital-asset-rpc-infrastructure/target/ - key: cargo-${{ hashFiles('**/Cargo.lock') }}-0001 + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + digital-asset-rpc-infrastructure/target/ + key: cargo-${{ hashFiles('**/Cargo.lock') }}-0001 # Cargo.lock - name: Check lock file run: | cargo tree git checkout Cargo.lock - cargo tree --frozen + cargo tree # fmt - name: Check fmt diff --git a/Api.Dockerfile b/Api.Dockerfile index cbc4ee6ee..a0b55a29f 100644 --- a/Api.Dockerfile +++ b/Api.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Builder.Dockerfile b/Builder.Dockerfile index 6b2b4b1bb..e883c7c8a 100644 --- a/Builder.Dockerfile +++ b/Builder.Dockerfile @@ -1,11 +1,12 @@ -FROM rust:1.76-bullseye AS builder +FROM rust:1.79-bullseye AS builder RUN apt-get update -y && \ - apt-get install -y build-essential make git - + apt-get install -y build-essential make git + RUN mkdir /rust RUN mkdir /rust/bins COPY Cargo.toml /rust COPY core /rust/core +COPY backfill /rust/backfill COPY das_api /rust/das_api COPY digital_asset_types /rust/digital_asset_types COPY integration_tests /rust/integration_tests @@ -19,7 +20,7 @@ COPY blockbuster rust/blockbuster WORKDIR /rust RUN --mount=type=cache,target=/rust/target,id=das-rust \ cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins - -FROM rust:1.75-slim-bullseye as final + +FROM rust:1.79-slim-bullseye as final COPY --from=builder /rust/bins /das/ CMD echo "Built the DAS API bins!" diff --git a/Cargo.lock b/Cargo.lock index 2e8cba4d8..5b7085c0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,8 +18,8 @@ version = "0.7.12" dependencies = [ "anyhow", "bs58 0.4.0", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.26", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -42,18 +42,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -93,11 +93,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -109,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "anchor-attribute-access-control" @@ -170,7 +170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f2a3e1df4685f18d12a943a9f2a7456305401af21a07c9fe076ef9ecd6e400" dependencies = [ "anchor-syn", - "bs58 0.5.0", + "bs58 0.5.1", "proc-macro2", "quote", "syn 1.0.109", @@ -235,7 +235,7 @@ dependencies = [ "solana-account-decoder", "solana-client", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tokio", "url", ] @@ -258,7 +258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4e2e5be518ec6053d90a2a7f26843dbee607583c779e6c8395951b9739bdfbe" dependencies = [ "anchor-syn", - "borsh-derive-internal 0.10.3", + "borsh-derive-internal 0.10.4", "proc-macro2", "quote", "syn 1.0.109", @@ -293,11 +293,11 @@ dependencies = [ "arrayref", "base64 0.13.1", "bincode", - "borsh 0.10.3", + "borsh 0.10.4", "bytemuck", - "getrandom 0.2.10", - "solana-program", - "thiserror", + "getrandom 0.2.15", + "solana-program 1.18.22", + "thiserror 1.0.69", ] [[package]] @@ -307,7 +307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9101b84702fed2ea57bd22992f75065da5648017135b844283a2f6d74f27825" dependencies = [ "anyhow", - "bs58 0.5.0", + "bs58 0.5.1", "heck 0.3.3", "proc-macro2", "quote", @@ -315,7 +315,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "syn 1.0.109", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -344,63 +344,65 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "once_cell", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "ark-bn254" @@ -443,7 +445,7 @@ dependencies = [ "derivative", "digest 0.10.7", "itertools 0.10.5", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-traits", "paste", "rustc_version", @@ -466,7 +468,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-traits", "proc-macro2", "quote", @@ -495,7 +497,7 @@ dependencies = [ "ark-serialize-derive", "ark-std", "digest 0.10.7", - "num-bigint 0.4.4", + "num-bigint 0.4.6", ] [[package]] @@ -521,15 +523,15 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ascii" @@ -549,8 +551,8 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror", - "time 0.3.29", + "thiserror 1.0.69", + "time 0.3.37", ] [[package]] @@ -562,7 +564,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "synstructure", + "synstructure 0.12.6", ] [[package]] @@ -605,12 +607,11 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 3.1.0", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -618,9 +619,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "brotli", "flate2", @@ -632,89 +633,59 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.7.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" dependencies = [ - "async-lock 3.1.1", "async-task", "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.0.1", + "fastrand", + "futures-lite", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 1.9.0", + "async-channel 2.3.1", "async-executor", - "async-io 1.13.0", - "async-lock 2.8.0", + "async-io", + "async-lock", "blocking", - "futures-lite 1.13.0", + "futures-lite", "once_cell", "tokio", ] [[package]] name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2 0.4.9", - "waker-fn", -] - -[[package]] -name = "async-io" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" dependencies = [ - "async-lock 3.1.1", + "async-lock", "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.0.1", + "futures-lite", "parking", - "polling 3.4.0", - "rustix 0.38.18", + "polling", + "rustix", "slab", "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", + "windows-sys 0.59.0", ] [[package]] name = "async-lock" -version = "3.1.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 3.1.0", + "event-listener 5.4.0", "event-listener-strategy", "pin-project-lite", ] @@ -730,20 +701,20 @@ dependencies = [ [[package]] name = "async-std" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +checksum = "c634475f29802fde2b8f0b505b1bd00dfe4df7d4a000f0b36f7671197d5c3615" dependencies = [ "async-attributes", "async-channel 1.9.0", "async-global-executor", - "async-io 1.13.0", - "async-lock 2.8.0", + "async-io", + "async-lock", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite 1.13.0", + "futures-lite", "gloo-timers", "kv-log-macro", "log", @@ -757,9 +728,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -768,30 +739,30 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "async-task" -version = "4.5.0" +version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -831,17 +802,71 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] [[package]] name = "backon" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ - "fastrand 1.9.0", + "fastrand", "futures-core", "pin-project", "tokio", @@ -849,17 +874,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -895,9 +920,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" @@ -919,7 +944,7 @@ name = "bgtask_creator" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", + "clap 4.5.26", "digital_asset_types", "futures", "lazy_static", @@ -951,9 +976,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" dependencies = [ "serde", ] @@ -981,9 +1006,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec", @@ -1024,7 +1049,7 @@ version = "2.3.0" dependencies = [ "anchor-lang", "async-trait", - "borsh 0.10.3", + "borsh 0.10.4", "bs58 0.4.0", "bytemuck", "flatbuffers", @@ -1052,23 +1077,20 @@ dependencies = [ "spl-token-2022", "spl-token-group-interface", "spl-token-metadata-interface", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "blocking" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ - "async-channel 2.1.0", - "async-lock 3.1.1", + "async-channel 2.3.1", "async-task", - "fastrand 2.0.1", "futures-io", - "futures-lite 2.0.1", + "futures-lite", "piper", - "tracing", ] [[package]] @@ -1083,21 +1105,21 @@ dependencies = [ [[package]] name = "borsh" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +checksum = "115e54d64eb62cdebad391c19efc9dce4981c690c85a33a12199d99bb9546fee" dependencies = [ - "borsh-derive 0.10.3", + "borsh-derive 0.10.4", "hashbrown 0.13.2", ] [[package]] name = "borsh" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +checksum = "9fb65153674e51d3a42c8f27b05b9508cea85edfaade8aa46bc8fc18cecdfef3" dependencies = [ - "borsh-derive 1.5.1", + "borsh-derive 1.5.4", "cfg_aliases", ] @@ -1116,12 +1138,12 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +checksum = "831213f80d9423998dd696e2c5345aba6be7a0bd8cd19e31c5243e13df1cef89" dependencies = [ - "borsh-derive-internal 0.10.3", - "borsh-schema-derive-internal 0.10.3", + "borsh-derive-internal 0.10.4", + "borsh-schema-derive-internal 0.10.4", "proc-macro-crate 0.1.5", "proc-macro2", "syn 1.0.109", @@ -1129,16 +1151,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +checksum = "a396e17ad94059c650db3d253bb6e25927f1eb462eede7e7a153bb6e75dce0a7" dependencies = [ "once_cell", "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.66", - "syn_derive", + "syn 2.0.96", ] [[package]] @@ -1154,9 +1175,9 @@ dependencies = [ [[package]] name = "borsh-derive-internal" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +checksum = "65d6ba50644c98714aa2a70d13d7df3cd75cd2b523a2b452bf010443800976b3" dependencies = [ "proc-macro2", "quote", @@ -1176,9 +1197,9 @@ dependencies = [ [[package]] name = "borsh-schema-derive-internal" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +checksum = "276691d96f063427be83e6692b86148e488ebba9f48f77788724ca027ba3b6d4" dependencies = [ "proc-macro2", "quote", @@ -1187,9 +1208,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1198,9 +1219,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1214,18 +1235,18 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", ] [[package]] name = "bstr" -version = "1.8.0" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "serde", @@ -1233,9 +1254,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bv" @@ -1249,9 +1270,9 @@ dependencies = [ [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1260,9 +1281,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", @@ -1271,22 +1292,22 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.5.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -1297,9 +1318,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cadence" @@ -1326,17 +1347,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" dependencies = [ "libc", - "thiserror", + "thiserror 1.0.69", +] + +[[package]] +name = "cargo-lock" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72" +dependencies = [ + "semver", + "serde", + "toml 0.7.8", + "url", ] [[package]] name = "cc" -version = "1.0.83" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1353,9 +1387,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1363,7 +1397,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -1386,7 +1420,7 @@ dependencies = [ "bitflags 1.3.2", "strsim 0.8.0", "textwrap 0.11.0", - "unicode-width", + "unicode-width 0.1.14", "vec_map", ] @@ -1404,29 +1438,29 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.0", + "textwrap 0.16.1", ] [[package]] name = "clap" -version = "4.4.8" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", - "clap_derive 4.4.7", + "clap_derive 4.5.24", ] [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", - "strsim 0.10.0", + "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] @@ -1444,14 +1478,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -1465,15 +1499,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -1490,9 +1524,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", @@ -1509,19 +1543,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", + "portable-atomic 1.10.0", ] [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width", - "windows-sys 0.52.0", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] @@ -1552,15 +1587,21 @@ checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "convert_case" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1568,91 +1609,85 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1703,11 +1738,39 @@ dependencies = [ "zeroize", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rand_core 0.6.4", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "darling" -version = "0.20.3" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -1715,111 +1778,207 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.66", + "strsim 0.11.1", + "syn 2.0.96", ] [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] -name = "das-core" +name = "das-bubblegum" version = "0.7.2" dependencies = [ + "anchor-client", "anyhow", - "backon", - "cadence", - "cadence-macros", - "clap 4.4.8", - "figment", - "plerkle_messenger", - "solana-account-decoder", + "blockbuster", + "borsh 0.10.4", + "bs58 0.4.0", + "clap 4.5.26", + "das-core", + "digital_asset_types", + "futures", + "heck 0.5.0", + "log", + "mpl-bubblegum", + "num-traits", + "program_transformers", + "sea-orm", + "serde_json", + "sha3 0.10.8", "solana-client", + "solana-program 1.18.22", "solana-sdk", "solana-transaction-status", + "spl-account-compression", + "spl-token", "sqlx", - "thiserror", + "thiserror 1.0.69", "tokio", + "tracing", ] [[package]] -name = "das-ops" +name = "das-core" version = "0.7.2" dependencies = [ - "anchor-client", "anyhow", "backon", - "borsh 0.10.3", + "borsh 0.10.4", + "bs58 0.4.0", "cadence", "cadence-macros", - "clap 4.4.8", - "das-core", + "clap 4.5.26", + "derive_more", "digital_asset_types", - "env_logger 0.10.0", "figment", - "flatbuffers", "futures", "indicatif", "log", - "mpl-bubblegum", "plerkle_messenger", - "plerkle_serialization", + "reqwest", "sea-orm", + "serde", + "serde_json", "solana-account-decoder", "solana-client", "solana-sdk", "solana-transaction-status", "spl-account-compression", - "thiserror", + "sqlx", + "thiserror 1.0.69", "tokio", + "url", ] [[package]] -name = "das_api" +name = "das-grpc-ingest" version = "0.7.2" dependencies = [ - "anchor-lang", - "async-trait", - "blockbuster", - "bs58 0.4.0", - "cadence", - "cadence-macros", + "anyhow", + "async-stream", + "atty", + "cargo-lock", + "chrono", + "clap 4.5.26", + "das-bubblegum", + "das-core", "digital_asset_types", - "env_logger 0.10.0", - "figment", + "futures", + "git-version", + "hex", "hyper", - "jsonrpsee", - "jsonrpsee-core", - "log", - "metrics", - "mpl-bubblegum", - "mpl-token-metadata", - "open-rpc-derive", - "open-rpc-schema", - "schemars", + "json5", + "lazy_static", + "lru", + "opentelemetry", + "opentelemetry-jaeger", + "opentelemetry_sdk", + "program_transformers", + "prometheus", + "redis 0.25.4", + "reqwest", + "rust-crypto", + "sea-orm", + "serde", + "serde_json", + "serde_yaml", + "solana-sdk", + "sqlx", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "vergen", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", + "yellowstone-grpc-tools", +] + +[[package]] +name = "das-ops" +version = "0.7.2" +dependencies = [ + "anchor-client", + "anyhow", + "borsh 0.10.4", + "bs58 0.4.0", + "cadence", + "cadence-macros", + "clap 4.5.26", + "das-bubblegum", + "das-core", + "digital_asset_types", + "env_logger 0.10.2", + "figment", + "futures", + "indicatif", + "log", + "mpl-bubblegum", + "mpl-token-metadata", + "program_transformers", + "reqwest", + "sea-orm", + "serde_json", + "solana-account-decoder", + "solana-client", + "solana-program 1.18.22", + "solana-sdk", + "solana-transaction-status", + "spl-account-compression", + "sqlx", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "das_api" +version = "0.7.2" +dependencies = [ + "anchor-lang", + "async-trait", + "blockbuster", + "bs58 0.4.0", + "cadence", + "cadence-macros", + "digital_asset_types", + "env_logger 0.10.2", + "figment", + "hyper", + "jsonrpsee", + "jsonrpsee-core", + "log", + "metrics", + "mpl-bubblegum", + "mpl-token-metadata", + "open-rpc-derive", + "open-rpc-schema", + "schemars", "schemars_derive", "sea-orm", "serde", "serde_json", "solana-sdk", "sqlx", - "thiserror", + "thiserror 1.0.69", "tokio", "tower", "tower-http", @@ -1833,17 +1992,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.10", ] [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "der" @@ -1863,17 +2022,18 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-traits", "rusticata-macros", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -1894,11 +2054,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.96", +] + [[package]] name = "deunicode" -version = "1.4.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e854126756c496b8c81dec88f9a706b15b875c5849d4097a3854476b9fdf94" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" [[package]] name = "dialoguer" @@ -1938,8 +2111,8 @@ version = "0.7.2" dependencies = [ "async-trait", "blockbuster", - "borsh 0.10.3", - "borsh-derive 0.10.3", + "borsh 0.10.4", + "borsh-derive 0.10.4", "bs58 0.4.0", "futures", "indexmap 1.9.3", @@ -1957,7 +2130,7 @@ dependencies = [ "serde_json", "solana-sdk", "spl-concurrent-merkle-tree", - "thiserror", + "thiserror 1.0.69", "tokio", "url", ] @@ -1984,13 +2157,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -2013,7 +2186,7 @@ checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -2024,9 +2197,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "eager" @@ -2049,7 +2222,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 3.2.1", "ed25519", "rand 0.7.3", "serde", @@ -2071,24 +2244,24 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" dependencies = [ "serde", ] [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -2110,7 +2283,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -2128,9 +2301,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -2147,12 +2320,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -2163,43 +2336,32 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "5.3.0" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", "pin-project-lite", - "portable-atomic 1.6.0", + "portable-atomic 1.10.0", "portable-atomic-util", ] [[package]] name = "event-listener-strategy" -version = "0.3.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 3.1.0", + "event-listener 5.4.0", "pin-project-lite", ] [[package]] name = "fake" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25829bde82205da46e1823b2259db6273379f626fc211f126f65654a2669be" +checksum = "2d391ba4af7f1d93f01fcf7b2f29e2bc9348e109dfdbf4dcbdc51dfa38dab0b6" dependencies = [ "deunicode", "rand 0.8.5", @@ -2207,18 +2369,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "feature-probe" @@ -2232,8 +2385,8 @@ version = "0.7.12" dependencies = [ "anyhow", "async-trait", - "borsh 0.10.3", - "clap 4.4.8", + "borsh 0.10.4", + "clap 4.5.26", "mpl-bubblegum", "solana-account-decoder", "solana-client", @@ -2242,26 +2395,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "figment" -version = "0.10.11" +version = "0.10.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a014ac935975a70ad13a3bff2463b1c1b083b35ae4cb6309cfc59476aa7a181f" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", "pear", "serde", "serde_yaml", - "toml 0.8.8", + "toml 0.8.19", "uncased", "version_check", ] [[package]] -name = "finl_unicode" -version = "1.2.0" +name = "five8_const" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b4f62f0f8ca357f93ae90c8c2dd1041a1f665fde2f889ea9b1787903829015" +dependencies = [ + "five8_core", +] + +[[package]] +name = "five8_core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94474d15a76982be62ca8a39570dccce148d98c238ebb7408b0a21b2c4bdddc4" + +[[package]] +name = "fixedbitset" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" @@ -2275,9 +2449,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -2289,6 +2463,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2401,29 +2581,13 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.0.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ - "fastrand 2.0.1", + "fastrand", "futures-core", "futures-io", - "memchr", "parking", "pin-project-lite", ] @@ -2436,7 +2600,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -2511,9 +2675,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "js-sys", @@ -2524,28 +2688,54 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "glob" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "globset" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" dependencies = [ "aho-corasick", "bstr", - "fnv", "log", - "regex", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] name = "gloo-timers" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" dependencies = [ "futures-channel", "futures-core", @@ -2566,9 +2756,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -2576,7 +2766,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2598,7 +2788,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2607,7 +2797,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", ] [[package]] @@ -2621,9 +2811,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", "allocator-api2", @@ -2631,9 +2821,14 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hashlink" @@ -2641,7 +2836,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.1", + "hashbrown 0.14.5", ] [[package]] @@ -2679,9 +2874,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" [[package]] name = "hex" @@ -2697,9 +2898,9 @@ checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac 0.12.1", ] @@ -2736,9 +2937,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2747,9 +2948,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2764,9 +2965,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2782,9 +2983,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -2797,7 +2998,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio", "tower-service", "tracing", @@ -2806,9 +3007,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", @@ -2818,6 +3019,18 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2833,16 +3046,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -2855,107 +3068,241 @@ dependencies = [ ] [[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.5.0" +name = "icu_collections" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "displaydoc", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "im" -version = "15.1.0" +name = "icu_locid" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "rayon", - "serde", - "sized-chunks", - "typenum", - "version_check", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "indexmap" -version = "1.9.3" +name = "icu_locid_transform" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", ] [[package]] -name = "indexmap" -version = "2.6.0" +name = "icu_locid_transform_data" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" -dependencies = [ - "equivalent", - "hashbrown 0.15.0", - "serde", -] +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" [[package]] -name = "indicatif" -version = "0.17.8" +name = "icu_normalizer" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic 1.6.0", - "unicode-width", + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", ] [[package]] -name = "inlinable_string" -version = "0.1.15" +name = "icu_normalizer_data" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" [[package]] -name = "insta" -version = "1.35.1" +name = "icu_properties" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ - "console", - "lazy_static", - "linked-hash-map", - "serde", - "similar", - "yaml-rust", + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", ] [[package]] -name = "instant" -version = "0.1.12" +name = "icu_properties_data" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "rayon", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +dependencies = [ + "console", + "number_prefix", + "portable-atomic 1.10.0", + "unicode-width 0.2.0", + "web-time 1.1.0", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "insta" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "serde", + "similar", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "integration_tests" version = "0.1.0" dependencies = [ "anyhow", - "borsh 0.10.3", + "borsh 0.10.4", "das_api", "digital_asset_types", "flatbuffers", @@ -2984,34 +3331,29 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.3", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ipnet" -version = "2.8.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.18", - "windows-sys 0.48.0", + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -3023,37 +3365,49 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonpath_lib" version = "0.3.0" @@ -3108,13 +3462,13 @@ dependencies = [ "globset", "hyper", "jsonrpsee-types", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "rand 0.8.5", "rustc-hash", "serde", "serde_json", "soketto", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -3164,24 +3518,24 @@ dependencies = [ "beef", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "tracing", ] [[package]] name = "kaigan" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a26f49495f94a283312e7ef45a243540ef20c9356bb01c8d84a61ac8ba5339b" +checksum = "b6dd100976df9dd59d0c3fecf6f9ad3f161a087374d1b2a77ebb4ad8920f11bb" dependencies = [ - "borsh 0.10.3", + "borsh 0.10.4", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] @@ -3197,15 +3551,25 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.8.0", + "libc", +] [[package]] name = "libsecp256k1" @@ -3263,8 +3627,8 @@ checksum = "3c9a85a9752c549ceb7578064b4ed891179d20acd85f27318573b64d2d7ee7ee" dependencies = [ "ark-bn254", "ark-ff", - "num-bigint 0.4.4", - "thiserror", + "num-bigint 0.4.6", + "thiserror 1.0.69", ] [[package]] @@ -3275,15 +3639,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] -name = "linux-raw-sys" -version = "0.4.10" +name = "litemap" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "load_generation" @@ -3293,7 +3657,7 @@ dependencies = [ "mpl-token-metadata", "rand 0.8.5", "solana-client", - "solana-program", + "solana-program 1.18.22", "solana-sdk", "spl-associated-token-account", "spl-token", @@ -3302,9 +3666,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -3312,13 +3676,22 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" dependencies = [ "value-bag", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "matchers" version = "0.1.0" @@ -3328,6 +3701,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -3340,9 +3719,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -3364,9 +3743,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -3401,7 +3780,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "metrics-macros", "portable-atomic 0.3.20", ] @@ -3435,9 +3814,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -3451,22 +3830,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.8" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3499,41 +3878,41 @@ dependencies = [ "anchor-lang", "bytemuck", "mpl-noop", - "solana-program", + "solana-program 1.18.22", "spl-concurrent-merkle-tree", ] [[package]] name = "mpl-bubblegum" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3cbca5deb859e66a1a21ada94f2eaab3eb5caa4584c0c8ade0efac29a5414b8" +checksum = "a9eff5ae5cafd1acdf7e7c93359da1eec91dcaede318470d9f68b78e8b7469f4" dependencies = [ - "borsh 0.10.3", + "borsh 0.10.4", "kaigan", "num-derive 0.3.3", "num-traits", - "solana-program", - "thiserror", + "solana-program 1.18.22", + "thiserror 1.0.69", ] [[package]] name = "mpl-core" -version = "0.8.0-beta.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1178d8a405352e2a2478abf5b62e2a4d6a91ed6f0470307ab6732614662dbf85" +checksum = "299e6db3f7d9ab8fba383e3c6e88f8869063e694cbeb56da8f1bdd355e255f8f" dependencies = [ - "base64 0.22.0", - "borsh 0.10.3", + "base64 0.22.1", + "borsh 0.10.4", "modular-bitfield", "num-derive 0.3.3", "num-traits", "rmp-serde", "serde", "serde_json", - "serde_with 3.6.1", - "solana-program", - "thiserror", + "serde_with 3.12.0", + "solana-program 1.18.22", + "thiserror 1.0.69", ] [[package]] @@ -3542,31 +3921,36 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "179556a9254920ca8b150b18728d2f3106879f710c1ef5a049a8f0d5b03eede5" dependencies = [ - "solana-program", + "solana-program 1.18.22", ] [[package]] name = "mpl-token-metadata" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b2de608098eb2ef2a5392069dea83084967e25a4d69d0380a6bb02454fc0fe" +checksum = "caf0f61b553e424a6234af1268456972ee66c2222e1da89079242251fa7479e5" dependencies = [ - "borsh 0.10.3", + "borsh 0.10.4", "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", - "solana-program", - "thiserror", + "serde_with 3.12.0", + "solana-program 1.18.22", + "thiserror 1.0.69", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -3584,12 +3968,13 @@ version = "0.7.2" dependencies = [ "async-trait", "blockbuster", - "borsh 0.10.3", + "borsh 0.10.4", "bs58 0.4.0", "cadence", "cadence-macros", "chrono", - "clap 4.4.8", + "clap 4.5.26", + "das-core", "digital_asset_types", "figment", "flatbuffers", @@ -3612,7 +3997,7 @@ dependencies = [ "spl-account-compression", "sqlx", "stretto", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing-subscriber", "url", @@ -3678,11 +4063,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -3697,6 +4081,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -3710,30 +4100,29 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -3754,9 +4143,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -3767,7 +4156,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", ] @@ -3782,11 +4171,11 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ - "num_enum_derive 0.7.2", + "num_enum_derive 0.7.3", ] [[package]] @@ -3798,19 +4187,28 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "num_enum_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", ] [[package]] @@ -3821,9 +4219,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -3839,15 +4237,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open-rpc-derive" @@ -3875,11 +4273,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -3896,7 +4294,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -3907,9 +4305,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -3917,6 +4315,87 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.7.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror 1.0.69", + "urlencoding", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +dependencies = [ + "async-trait", + "futures-core", + "futures-util", + "opentelemetry", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float 4.6.0", + "percent-encoding", + "rand 0.8.5", + "thiserror 1.0.69", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -3954,9 +4433,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -3971,12 +4450,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.10", ] [[package]] @@ -3995,22 +4474,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.5.8", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" @@ -4032,9 +4511,9 @@ dependencies = [ [[package]] name = "pear" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a386cd715229d399604b50d1361683fe687066f42d56f54be995bc6868f71c" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", @@ -4043,14 +4522,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f0f13dac8069c139e8300a6510e3f4143ecf5259c60b116a9b271b4ca0d54" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -4077,31 +4556,86 @@ dependencies = [ "num", ] +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.11", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "pest_meta" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.8", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.7.0", +] + [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -4111,12 +4645,12 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" dependencies = [ "atomic-waker", - "fastrand 2.0.1", + "fastrand", "futures-io", ] @@ -4133,9 +4667,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "plain" @@ -4145,9 +4679,9 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plerkle_messenger" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f8e4e8975dcd2dbf94c7f84409096f0a0e5af287eabc9a212704e9e325ec84" +checksum = "b5875d1318a10256692c1ad2ac50367b05bfa23e5107e82f21dd99e768e71ea8" dependencies = [ "async-mutex", "async-trait", @@ -4157,9 +4691,9 @@ dependencies = [ "figment", "futures", "log", - "redis", + "redis 0.22.3", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -4174,37 +4708,22 @@ dependencies = [ "serde", "solana-sdk", "solana-transaction-status", - "thiserror", -] - -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", + "thiserror 1.0.69", ] [[package]] name = "polling" -version = "3.4.0" +version = "3.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30054e72317ab98eddd8561db0f6524df3367636884b7b21b703e4b280a84a14" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", + "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.18", + "rustix", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4225,29 +4744,48 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" dependencies = [ - "portable-atomic 1.6.0", + "portable-atomic 1.10.0", ] [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "portable-atomic-util" -version = "0.1.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a7411625b38d51b41421c6333976adffd4674a925a978856734a2dc853449b" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" dependencies = [ - "portable-atomic 1.6.0", + "portable-atomic 1.10.0", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" +dependencies = [ + "proc-macro2", + "syn 2.0.96", +] [[package]] name = "proc-macro-crate" @@ -4303,9 +4841,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.83" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -4318,7 +4856,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", "version_check", "yansi", ] @@ -4331,12 +4869,14 @@ dependencies = [ "bs58 0.4.0", "cadence", "cadence-macros", + "das-core", "digital_asset_types", "futures", "heck 0.5.0", "mpl-bubblegum", "num-traits", "sea-orm", + "serde", "serde_json", "solana-sdk", "solana-transaction-status", @@ -4344,24 +4884,83 @@ dependencies = [ "spl-token", "spl-token-2022", "sqlx", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] +[[package]] +name = "project-root" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bccbff07d5ed689c4087d20d7307a52ab6141edeedf487c3876a55b86cf63df" + [[package]] name = "prometheus" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ "cfg-if", "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "protobuf", - "thiserror", + "thiserror 1.0.69", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.96", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", ] [[package]] @@ -4370,13 +4969,22 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + [[package]] name = "proxy-wasm" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823b744520cd4a54ba7ebacbffe4562e839d6dcd8f89209f96a1ace4f5229cd4" +checksum = "14a5a4df5a1ab77235e36a0a0f638687ee1586d21ee9774037693001e94d4e11" dependencies = [ - "hashbrown 0.13.2", + "hashbrown 0.14.5", "log", ] @@ -4417,7 +5025,7 @@ checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -4432,7 +5040,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.21.12", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -4450,7 +5058,7 @@ dependencies = [ "rustls 0.21.12", "rustls-native-certs", "slab", - "thiserror", + "thiserror 1.0.69", "tinyvec", "tracing", ] @@ -4463,16 +5071,16 @@ checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ "bytes", "libc", - "socket2 0.5.7", + "socket2", "tracing", "windows-sys 0.48.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -4580,7 +5188,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.15", ] [[package]] @@ -4629,7 +5237,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.29", + "time 0.3.37", "yasna", ] @@ -4651,7 +5259,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes", - "combine 4.6.6", + "combine 4.6.7", "futures", "futures-util", "itoa", @@ -4666,6 +5274,29 @@ dependencies = [ "url", ] +[[package]] +name = "redis" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec" +dependencies = [ + "async-trait", + "bytes", + "combine 4.6.7", + "futures-util", + "itoa", + "native-tls", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-native-tls", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4677,34 +5308,34 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.8.0", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", - "thiserror", + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -4718,13 +5349,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.5", ] [[package]] @@ -4735,15 +5366,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rend" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] @@ -4790,7 +5421,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", "winreg", ] @@ -4811,26 +5442,28 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.2" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911b295d2d302948838c8ac142da1ee09fa7863163b44e6715bc9357905878b8" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "getrandom 0.2.10", + "cfg-if", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -4842,9 +5475,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" dependencies = [ "proc-macro2", "quote", @@ -4909,12 +5542,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.32.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", - "borsh 0.10.3", + "borsh 1.5.4", "bytes", "num-traits", "rand 0.8.5", @@ -4925,9 +5558,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -4937,15 +5570,15 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-serialize" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -4961,29 +5594,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.27" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" -dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", - "linux-raw-sys 0.4.10", - "windows-sys 0.48.0", + "linux-raw-sys", + "windows-sys 0.59.0", ] [[package]] @@ -5005,7 +5624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.2", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -5024,9 +5643,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64 0.21.7", ] @@ -5037,36 +5656,36 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.2", + "ring 0.17.8", "untrusted 0.9.0", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "dyn-clone", "schemars_derive", @@ -5076,14 +5695,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] @@ -5109,17 +5728,17 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -5143,8 +5762,8 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror", - "time 0.3.29", + "thiserror 1.0.69", + "time 0.3.37", "tracing", "url", "uuid", @@ -5205,7 +5824,7 @@ dependencies = [ "rust_decimal", "sea-query-derive 0.2.0", "serde_json", - "time 0.3.29", + "time 0.3.37", "uuid", ] @@ -5229,7 +5848,7 @@ dependencies = [ "sea-query 0.27.2", "serde_json", "sqlx", - "time 0.3.29", + "time 0.3.37", "uuid", ] @@ -5243,7 +5862,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -5256,7 +5875,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -5312,11 +5931,11 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.9.2" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -5325,9 +5944,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -5335,15 +5954,18 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +dependencies = [ + "serde", +] [[package]] name = "serde" -version = "1.0.202" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -5359,33 +5981,33 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -5394,9 +6016,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -5425,20 +6047,20 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.6.1", - "time 0.3.29", + "serde_with_macros 3.12.0", + "time 0.3.37", ] [[package]] @@ -5450,28 +6072,28 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -5488,7 +6110,7 @@ dependencies = [ "futures", "lazy_static", "log", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "serial_test_derive", ] @@ -5500,7 +6122,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -5529,9 +6151,9 @@ dependencies = [ [[package]] name = "sha1_smol" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sha2" @@ -5594,11 +6216,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -5611,15 +6239,15 @@ checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" [[package]] name = "siphasher" @@ -5654,19 +6282,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -5690,9 +6308,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b109fd3a106e079005167e5b0e6f6d2c88bbedec32530837b584791a8b5abf36" +checksum = "b4185d569c062983fc2a618ae4ee6fe1a139b36bce7a25045647c49bf0020a53" dependencies = [ "Inflector", "base64 0.21.7", @@ -5709,22 +6327,65 @@ dependencies = [ "spl-token-2022", "spl-token-group-interface", "spl-token-metadata-interface", - "thiserror", + "thiserror 1.0.69", "zstd", ] +[[package]] +name = "solana-account-info" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e053b991f91fd274df53e070c77a0a6a33681a5102c6421a0ca9ffaa0040368a" +dependencies = [ + "bincode", + "serde", + "solana-program-error", + "solana-program-memory", + "solana-pubkey", +] + +[[package]] +name = "solana-atomic-u64" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "966dce88672728380c476d5d3e54c02025875100b8246db05669961806c9575e" +dependencies = [ + "parking_lot 0.12.3", +] + +[[package]] +name = "solana-bincode" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117b9646b1e9e6c4b48f363ad4c5af25c4ab35754ff307714e5fec2c3c4bb6b" +dependencies = [ + "bincode", + "serde", + "solana-instruction", +] + +[[package]] +name = "solana-borsh" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c55b83c305eac62095b6f24ea2ae729f17de47e5a4e866ee4ddd0dc501b351e" +dependencies = [ + "borsh 0.10.4", + "borsh 1.5.4", +] + [[package]] name = "solana-clap-utils" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074ef478856a45d5627270fbc6b331f91de9aae7128242d9e423931013fb8a2a" +checksum = "c817832e71886dbea877d1aa911c9ce2e984a39081bb56ee30d4c835567827a6" dependencies = [ "chrono", "clap 2.34.0", "rpassword", "solana-remote-wallet", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tiny-bip39", "uriparse", "url", @@ -5732,16 +6393,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a9f32c42402c4b9484d5868ac74b7e0a746e3905d8bfd756e1203e50cbb87e" +checksum = "7fa9cc6e8e59adf70acbf5cac21342ae8b5e41cbf05519fe5f6287e84ab40f63" dependencies = [ "async-trait", "bincode", "dashmap", "futures", "futures-util", - "indexmap 2.6.0", + "indexmap 2.7.0", "indicatif", "log", "quinn", @@ -5759,15 +6420,27 @@ dependencies = [ "solana-thin-client", "solana-tpu-client", "solana-udp-client", - "thiserror", + "thiserror 1.0.69", "tokio", ] +[[package]] +name = "solana-clock" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2387b936492cab0649c2a3e3fcfb282077029b533fa8454c88c41dff3bc2552" +dependencies = [ + "serde", + "serde_derive", + "solana-sdk-macro 2.1.8", + "solana-sysvar-id", +] + [[package]] name = "solana-config-program" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d75b803860c0098e021a26f0624129007c15badd5b0bc2fbd9f0e1a73060d3b" +checksum = "d02fb29934427f1487d2149fe8bcb405306729b2f22a2ad616bb8ffd024cee7b" dependencies = [ "bincode", "chrono", @@ -5779,15 +6452,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9306ede13e8ceeab8a096bcf5fa7126731e44c201ca1721ea3c38d89bcd4111" +checksum = "d8e5a2e26448b3e04ce673794994ff27f3972ec8a806c224eccc02e09f751ca5" dependencies = [ "async-trait", "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.6.0", + "indexmap 2.7.0", "log", "rand 0.8.5", "rayon", @@ -5795,64 +6468,164 @@ dependencies = [ "solana-measure", "solana-metrics", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tokio", ] [[package]] -name = "solana-frozen-abi" -version = "1.18.26" +name = "solana-cpi" +version = "2.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ab2c30c15311b511c0d1151e4ab6bc9a3e080a37e7c6e7c2d96f5784cf9434" +checksum = "00bae0591481827ac9cfce5573aad2918bb01f91289b811ea531df4fcb73d136" dependencies = [ - "block-buffer 0.10.4", - "bs58 0.4.0", - "bv", - "either", - "generic-array", - "im", - "lazy_static", - "log", - "memmap2", - "rustc_version", + "solana-account-info", + "solana-define-syscall", + "solana-instruction", + "solana-program-error", + "solana-pubkey", + "solana-stable-layout", +] + +[[package]] +name = "solana-decode-error" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8880dc18fb97c6205214d1f3ce2f1152e997ecc6f6da4bb458fbf6e6207a0693" +dependencies = [ + "num-traits", +] + +[[package]] +name = "solana-define-syscall" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6452c4a8fc77cc60ad2934b19f2d75691067f17355b34462d52285395c1c99db" + +[[package]] +name = "solana-epoch-schedule" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e783a735416c534228f24f18f18ade0b189c2c8a93be486c9a26bd314517d93" +dependencies = [ + "serde", + "serde_derive", + "solana-sdk-macro 2.1.8", + "solana-sysvar-id", +] + +[[package]] +name = "solana-fee-calculator" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fa18582732d94369263c42eeee967ff919e99b9b15ba747fb7534aa24fbbc0" +dependencies = [ + "log", + "serde", + "serde_derive", +] + +[[package]] +name = "solana-frozen-abi" +version = "1.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20a6ef2db80dceb124b7bf81cca3300804bf427d2711973fc3df450ed7dfb26d" +dependencies = [ + "block-buffer 0.10.4", + "bs58 0.4.0", + "bv", + "either", + "generic-array", + "im", + "lazy_static", + "log", + "memmap2", + "rustc_version", "serde", "serde_bytes", "serde_derive", "sha2 0.10.8", "solana-frozen-abi-macro", "subtle", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "solana-frozen-abi-macro" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c142f779c3633ac83c84d04ff06c70e1f558c876f13358bed77ba629c7417932" +checksum = "70088de7d4067d19a7455609e2b393e6086bd847bb39c4d2bf234fc14827ef9e" dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "solana-geyser-plugin-interface" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4023dfd15b33053dd7193f58a267345d776601c3c7e37ad667e2ba43c7b8fadd" +checksum = "63f1835fe954e305097c83b4ce8548e675647eef6a3ea9ec13dbfe2d6b869398" dependencies = [ "log", "solana-sdk", "solana-transaction-status", - "thiserror", + "thiserror 1.0.69", +] + +[[package]] +name = "solana-hash" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e35f984e3d60a58184743446250cf724afb34ed65f794da0dc4b462f9c1929" +dependencies = [ + "borsh 1.5.4", + "bs58 0.5.1", + "bytemuck", + "bytemuck_derive", + "js-sys", + "serde", + "serde_derive", + "solana-atomic-u64", + "solana-sanitize", + "wasm-bindgen", +] + +[[package]] +name = "solana-instruction" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fc69f7f75df0b11e99c03393b24a7443aec0430518054de14715c59cfa716d" +dependencies = [ + "bincode", + "borsh 1.5.4", + "getrandom 0.2.15", + "js-sys", + "num-traits", + "serde", + "serde_derive", + "solana-define-syscall", + "solana-pubkey", + "wasm-bindgen", +] + +[[package]] +name = "solana-last-restart-slot" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fee98cc25000ee8bab1a4f63c7516d9521bc8a9747d8287ebb05e5d9b1d32ee1" +dependencies = [ + "serde", + "serde_derive", + "solana-sdk-macro 2.1.8", + "solana-sysvar-id", ] [[package]] name = "solana-logger" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121d36ffb3c6b958763312cbc697fbccba46ee837d3a0aa4fc0e90fcb3b884f3" +checksum = "b129da15193f26db62d62ae6bb9f72361f361bcdc36054be3ab8bc04cc7a4f31" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5861,9 +6634,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c01a7f9cdc9d9d37a3d5651b2fe7ec9d433c2a3470b9f35897e373b421f0737" +checksum = "6d195b73093a4964ba6b5943418054a5fcbba23eafdd0842fd973fcceac1a967" dependencies = [ "log", "solana-sdk", @@ -5871,9 +6644,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e36052aff6be1536bdf6f737c6e69aca9dbb6a2f3f582e14ecb0ddc0cd66ce" +checksum = "fe7b06860ffbf4cf4714182e1b7eb00eb3ff0bcc9cff615d05e01e488923883c" dependencies = [ "crossbeam-channel", "gethostname", @@ -5881,14 +6654,29 @@ dependencies = [ "log", "reqwest", "solana-sdk", - "thiserror", + "thiserror 1.0.69", +] + +[[package]] +name = "solana-msg" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac7a109b0c7a0ed26c1fbf3b0fec8809b5d4c74b5d597f0252d45255fd0d309" +dependencies = [ + "solana-define-syscall", ] +[[package]] +name = "solana-native-token" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7246817ae265f5a67be25f32ee52267f1c2fe29767ab601ef03c5086bfc64992" + [[package]] name = "solana-net-utils" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a1f5c6be9c5b272866673741e1ebc64b2ea2118e5c6301babbce526fdfb15f4" +checksum = "9400b50b8439868a99b5fa2d961d74e37b7a6c1d5865759d0b1c906c2ad6b2a9" dependencies = [ "bincode", "clap 3.2.25", @@ -5898,7 +6686,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_derive", - "socket2 0.5.7", + "socket2", "solana-logger", "solana-sdk", "solana-version", @@ -5908,15 +6696,15 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28acaf22477566a0fbddd67249ea5d859b39bacdb624aff3fadd3c5745e2643c" +checksum = "b01a386e852df67031195094628851b8d239dd71fe17b721c3993277e68cb3ab" dependencies = [ "ahash 0.8.11", "bincode", "bv", "caps", - "curve25519-dalek", + "curve25519-dalek 3.2.1", "dlopen2", "fnv", "lazy_static", @@ -5937,9 +6725,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10f4588cefd716b24a1a40dd32c278e43a560ab8ce4de6b5805c9d113afdfa1" +checksum = "fb2b2c8babfae4cace1a25b6efa00418f3acd852cf55d7cecc0360d3c5050479" dependencies = [ "ark-bn254", "ark-ec", @@ -5947,19 +6735,19 @@ dependencies = [ "ark-serialize", "base64 0.21.7", "bincode", - "bitflags 2.6.0", + "bitflags 2.8.0", "blake3", - "borsh 0.10.3", + "borsh 0.10.4", "borsh 0.9.3", - "borsh 1.5.1", + "borsh 1.5.4", "bs58 0.4.0", "bv", "bytemuck", "cc", "console_error_panic_hook", "console_log", - "curve25519-dalek", - "getrandom 0.2.10", + "curve25519-dalek 3.2.1", + "getrandom 0.2.15", "itertools 0.10.5", "js-sys", "lazy_static", @@ -5967,11 +6755,11 @@ dependencies = [ "libsecp256k1", "light-poseidon", "log", - "memoffset 0.9.0", - "num-bigint 0.4.4", - "num-derive 0.4.1", + "memoffset 0.9.1", + "num-bigint 0.4.6", + "num-derive 0.4.2", "num-traits", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "rand 0.8.5", "rustc_version", "rustversion", @@ -5983,18 +6771,144 @@ dependencies = [ "sha3 0.10.8", "solana-frozen-abi", "solana-frozen-abi-macro", - "solana-sdk-macro", - "thiserror", + "solana-sdk-macro 1.18.22", + "thiserror 1.0.69", "tiny-bip39", "wasm-bindgen", "zeroize", ] +[[package]] +name = "solana-program" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb05f5ffadb039285ee82efd9a593e0873220f840f0eac7069d962f9eb29a407" +dependencies = [ + "base64 0.22.1", + "bincode", + "bitflags 2.8.0", + "blake3", + "borsh 0.10.4", + "borsh 1.5.4", + "bs58 0.5.1", + "bv", + "bytemuck", + "bytemuck_derive", + "console_error_panic_hook", + "console_log", + "curve25519-dalek 4.1.3", + "five8_const", + "getrandom 0.2.15", + "js-sys", + "lazy_static", + "log", + "memoffset 0.9.1", + "num-bigint 0.4.6", + "num-derive 0.4.2", + "num-traits", + "parking_lot 0.12.3", + "rand 0.8.5", + "serde", + "serde_bytes", + "serde_derive", + "sha2 0.10.8", + "sha3 0.10.8", + "solana-account-info", + "solana-atomic-u64", + "solana-bincode", + "solana-borsh", + "solana-clock", + "solana-cpi", + "solana-decode-error", + "solana-define-syscall", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-hash", + "solana-instruction", + "solana-last-restart-slot", + "solana-msg", + "solana-native-token", + "solana-program-entrypoint", + "solana-program-error", + "solana-program-memory", + "solana-program-option", + "solana-program-pack", + "solana-pubkey", + "solana-rent", + "solana-sanitize", + "solana-sdk-macro 2.1.8", + "solana-secp256k1-recover", + "solana-serde-varint", + "solana-serialize-utils", + "solana-sha256-hasher", + "solana-short-vec", + "solana-slot-hashes", + "solana-slot-history", + "solana-stable-layout", + "solana-sysvar-id", + "solana-transaction-error", + "thiserror 1.0.69", + "wasm-bindgen", +] + +[[package]] +name = "solana-program-entrypoint" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f6148e740c6deed55fe343355f0cb3ec158d221e11aa8bb93a392fa62c4137" +dependencies = [ + "solana-account-info", + "solana-msg", + "solana-program-error", + "solana-pubkey", +] + +[[package]] +name = "solana-program-error" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87e99e4299728f450194b6adf946dde512d79d82275b1c73f6faea7e9075cef" +dependencies = [ + "borsh 1.5.4", + "num-traits", + "serde", + "serde_derive", + "solana-decode-error", + "solana-instruction", + "solana-msg", + "solana-pubkey", +] + +[[package]] +name = "solana-program-memory" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3691cdd84c0a4753b484f468aac19e0943fab1e71705b21d00d561ac6eea6449" +dependencies = [ + "num-traits", + "solana-define-syscall", +] + +[[package]] +name = "solana-program-option" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e99a3e016363a95cdbe23aaa2a68578ffa2ce8e37c4a642962201af6376ffc37" + +[[package]] +name = "solana-program-pack" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eba980dec9d5403ea299a3cdf27cd794e6b1a188acc8c5e3ae7d067b629eb24" +dependencies = [ + "solana-program-error", +] + [[package]] name = "solana-program-runtime" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf0c3eab2a80f514289af1f422c121defb030937643c43b117959d6f1932fb5" +checksum = "0444f9440f4459d377c41470b2eb48b527def81f3052b7a121f6aa8c7350cc52" dependencies = [ "base64 0.21.7", "bincode", @@ -6003,7 +6917,7 @@ dependencies = [ "itertools 0.10.5", "libc", "log", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "percentage", "rand 0.8.5", @@ -6015,14 +6929,40 @@ dependencies = [ "solana-metrics", "solana-sdk", "solana_rbpf", - "thiserror", + "thiserror 1.0.69", +] + +[[package]] +name = "solana-pubkey" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dba2b19db8b73ab96b309b6d2a9f26386e45e2af3618a27b92389da9a3df1f1" +dependencies = [ + "borsh 0.10.4", + "borsh 1.5.4", + "bs58 0.5.1", + "bytemuck", + "bytemuck_derive", + "curve25519-dalek 4.1.3", + "five8_const", + "getrandom 0.2.15", + "js-sys", + "num-traits", + "serde", + "serde_derive", + "solana-atomic-u64", + "solana-decode-error", + "solana-define-syscall", + "solana-sanitize", + "solana-sha256-hasher", + "wasm-bindgen", ] [[package]] name = "solana-pubsub-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b064e76909d33821b80fdd826e6757251934a52958220c92639f634bea90366d" +checksum = "0ee4a39e41e789b6f100c97d9f40c1d08381bf6e3d0e351065e542091cddb039" dependencies = [ "crossbeam-channel", "futures-util", @@ -6035,7 +6975,7 @@ dependencies = [ "solana-account-decoder", "solana-rpc-client-api", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", "tokio-tungstenite", @@ -6045,9 +6985,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a90e40ee593f6e9ddd722d296df56743514ae804975a76d47e7afed4e3da244" +checksum = "baad755c76ee0aab8890f0ef873e61b8b3012c523d33bfa5b062fe9be8cef370" dependencies = [ "async-mutex", "async-trait", @@ -6066,15 +7006,15 @@ dependencies = [ "solana-rpc-client-api", "solana-sdk", "solana-streamer", - "thiserror", + "thiserror 1.0.69", "tokio", ] [[package]] name = "solana-rayon-threadlimit" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66468f9c014992167de10cc68aad6ac8919a8c8ff428dc88c0d2b4da8c02b8b7" +checksum = "c1c2a0ccb0be7ca79e8ff0d7c786bce586433a5687ffbea522453d0b41c4bf4a" dependencies = [ "lazy_static", "num_cpus", @@ -6082,28 +7022,40 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c191019f4d4f84281a6d0dd9a43181146b33019627fc394e42e08ade8976b431" +checksum = "3d042a812537e3507e1c163c7573fc04c96e12d3eba512e3fe74c7393229fa39" dependencies = [ "console", "dialoguer", "log", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "qstring", "semver", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "uriparse", ] +[[package]] +name = "solana-rent" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138b60a6683d14d63b4cee532d50afcb54999679b5c53013969fd51977455e14" +dependencies = [ + "serde", + "serde_derive", + "solana-sdk-macro 2.1.8", + "solana-sysvar-id", +] + [[package]] name = "solana-rpc-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ed4628e338077c195ddbf790693d410123d17dec0a319b5accb4aaee3fb15c" +checksum = "3c6f5560283bd0a6833d1bd816299785058a870fff51b0df399fdb3ce92c8484" dependencies = [ "async-trait", "base64 0.21.7", @@ -6127,9 +7079,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c913551faa4a1ae4bbfef6af19f3a5cf847285c05b4409e37c8993b3444229" +checksum = "2e4ca77f89caa9071acadb1eed19c28a6691fd63d0563ed927c96bf734cf1c9c" dependencies = [ "base64 0.21.7", "bs58 0.4.0", @@ -6144,33 +7096,39 @@ dependencies = [ "solana-transaction-status", "solana-version", "spl-token-2022", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a47b6bb1834e6141a799db62bbdcf80d17a7d58d7bc1684c614e01a7293d7cf" +checksum = "42a6ea9ad81d63f18fb8b3a9b39643cc43eaf909199d67037e724562301d1df7" dependencies = [ "clap 2.34.0", "solana-clap-utils", "solana-rpc-client", "solana-sdk", - "thiserror", + "thiserror 1.0.69", ] +[[package]] +name = "solana-sanitize" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f71b885b953e9157b66eaba9a34507f2f840712ef54f483725ba510ee1bd89" + [[package]] name = "solana-sdk" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "580ad66c2f7a4c3cb3244fe21440546bd500f5ecb955ad9826e92a78dded8009" +checksum = "b5e0f0def5c5af07f53d321cea7b104487b522cfff77c3cae3da361bfe956e9e" dependencies = [ "assert_matches", "base64 0.21.7", "bincode", - "bitflags 2.6.0", - "borsh 1.5.1", + "bitflags 2.8.0", + "borsh 1.5.4", "bs58 0.4.0", "bytemuck", "byteorder", @@ -6187,9 +7145,9 @@ dependencies = [ "libsecp256k1", "log", "memmap2", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", - "num_enum 0.7.2", + "num_enum 0.7.3", "pbkdf2 0.11.0", "qstring", "qualifier_attr", @@ -6208,24 +7166,47 @@ dependencies = [ "solana-frozen-abi", "solana-frozen-abi-macro", "solana-logger", - "solana-program", - "solana-sdk-macro", - "thiserror", + "solana-program 1.18.22", + "solana-sdk-macro 1.18.22", + "thiserror 1.0.69", "uriparse", "wasm-bindgen", ] [[package]] name = "solana-sdk-macro" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b75d0f193a27719257af19144fdaebec0415d1c9e9226ae4bd29b791be5e9bd" +checksum = "c55c196c8050834c391a34b58e3c9fd86b15452ef1feeeafa1dbeb9d2291dfec" dependencies = [ "bs58 0.4.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.96", +] + +[[package]] +name = "solana-sdk-macro" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f0b358f336ceac3827881915e5293f121c023cbd2150115046356c66898cb8" +dependencies = [ + "bs58 0.5.1", + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "solana-secp256k1-recover" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460c2e36586bcce843cdeaaf2364f3db7fbd9f266325e93d5e9af33f2605dd7d" +dependencies = [ + "libsecp256k1", + "solana-define-syscall", + "thiserror 1.0.69", ] [[package]] @@ -6234,18 +7215,92 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" +[[package]] +name = "solana-serde-varint" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98449030e53dcc2c4f160acab99b2bdb3e24ea8bff8ca6e71a6e539a54bf3d7" +dependencies = [ + "serde", +] + +[[package]] +name = "solana-serialize-utils" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d659aac218580fc3fb3e8350669db9bb01bc1bc849c90f0741cbfccb6663eb94" +dependencies = [ + "solana-instruction", + "solana-pubkey", + "solana-sanitize", +] + +[[package]] +name = "solana-sha256-hasher" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0db90ad6643d4d626f923159eaa876000c09f8c2e9aa7ff59b803e8328712582" +dependencies = [ + "sha2 0.10.8", + "solana-define-syscall", + "solana-hash", +] + +[[package]] +name = "solana-short-vec" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f7de721a6c50cb3a41e027a623496be39e45c452fbf897f657cd1f2f67dbbd" +dependencies = [ + "serde", +] + +[[package]] +name = "solana-slot-hashes" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3840867aa6d0fac65d3a4c1f14fff650a8e148732a16c06ebd8a2389d79d4745" +dependencies = [ + "serde", + "serde_derive", + "solana-hash", + "solana-sysvar-id", +] + +[[package]] +name = "solana-slot-history" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "101583a12fcce9b52f845b3c773f4ae6c3f4ca6a46177dadbd83e276baf82326" +dependencies = [ + "bv", + "serde", + "serde_derive", + "solana-sysvar-id", +] + +[[package]] +name = "solana-stable-layout" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b923e1c9e42b6c98b1786ca003af6a0366932f08d63432e984fcc394b7b5e" +dependencies = [ + "solana-instruction", + "solana-pubkey", +] + [[package]] name = "solana-streamer" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8476e41ad94fe492e8c06697ee35912cf3080aae0c9e9ac6430835256ccf056" +checksum = "749720d82c5f31f7ec326da1e0baac098201de70f0874719172a55309433b449" dependencies = [ "async-channel 1.9.0", "bytes", "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.6.0", + "indexmap 2.7.0", "itertools 0.10.5", "libc", "log", @@ -6262,16 +7317,25 @@ dependencies = [ "solana-metrics", "solana-perf", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tokio", "x509-parser", ] +[[package]] +name = "solana-sysvar-id" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59351de877a7cf0cea0e436424ecf4ea0c08c59ff01ef0575436972b920b818c" +dependencies = [ + "solana-pubkey", +] + [[package]] name = "solana-thin-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c02245d0d232430e79dc0d624aa42d50006097c3aec99ac82ac299eaa3a73f" +checksum = "84535de1253afb6ccc4ae6852eb013ca734c439a902ec5e4684b90ed649a37c2" dependencies = [ "bincode", "log", @@ -6284,14 +7348,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67251506ed03de15f1347b46636b45c47da6be75015b4a13f0620b21beb00566" +checksum = "3ff514462bb715aaea9bc5c0ee60f83ab3f91e04279337c6b07d054153b616dc" dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.6.0", + "indexmap 2.7.0", "indicatif", "log", "rayon", @@ -6302,20 +7366,30 @@ dependencies = [ "solana-rpc-client", "solana-rpc-client-api", "solana-sdk", - "thiserror", + "thiserror 1.0.69", "tokio", ] +[[package]] +name = "solana-transaction-error" +version = "2.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c3d2147cfaad2a5518b8e15621008699e28d32d6233cd7a6b27a506e01f1515" +dependencies = [ + "solana-instruction", + "solana-sanitize", +] + [[package]] name = "solana-transaction-status" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3d36db1b2ab2801afd5482aad9fb15ed7959f774c81a77299fdd0ddcf839d4" +checksum = "670e387049812d42bdc8fcc4ff75452ff3cb00657af979a90f55f6d37dba9dd9" dependencies = [ "Inflector", "base64 0.21.7", "bincode", - "borsh 0.10.3", + "borsh 0.10.4", "bs58 0.4.0", "lazy_static", "log", @@ -6328,29 +7402,29 @@ dependencies = [ "spl-memo", "spl-token", "spl-token-2022", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "solana-udp-client" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a754a3c2265eb02e0c35aeaca96643951f03cee6b376afe12e0cf8860ffccd1" +checksum = "11183dae826f942ebd0401712c8a52367a4a6312f1cd325f304cd9551226fc8b" dependencies = [ "async-trait", "solana-connection-cache", "solana-net-utils", "solana-sdk", "solana-streamer", - "thiserror", + "thiserror 1.0.69", "tokio", ] [[package]] name = "solana-version" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44776bd685cc02e67ba264384acc12ef2931d01d1a9f851cb8cdbd3ce455b9e" +checksum = "8e8d518e61ce22c812df23d9c61ab9bcbef4df3e3d3dcaa74a999625f11bcf07" dependencies = [ "log", "rustc_version", @@ -6364,13 +7438,13 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25810970c91feb579bd3f67dca215fce971522e42bfd59696af89c5dfebd997c" +checksum = "5743503143fb2259c41a973a78e9aeeb8e21f1b03543c3bb85449926ea692719" dependencies = [ "bincode", "log", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "rustc_version", "serde", @@ -6378,38 +7452,38 @@ dependencies = [ "solana-frozen-abi", "solana-frozen-abi-macro", "solana-metrics", - "solana-program", + "solana-program 1.18.22", "solana-program-runtime", "solana-sdk", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "solana-zk-token-sdk" -version = "1.18.26" +version = "1.18.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbdf4249b6dfcbba7d84e2b53313698043f60f8e22ce48286e6fbe8a17c8d16" +checksum = "57ee07fa523b4cfcff68de774db7aa87d2da2c4357155a90bacd9a0a0af70a99" dependencies = [ "aes-gcm-siv", "base64 0.21.7", "bincode", "bytemuck", "byteorder", - "curve25519-dalek", + "curve25519-dalek 3.2.1", "getrandom 0.1.16", "itertools 0.10.5", "lazy_static", "merlin", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", "rand 0.7.3", "serde", "serde_json", "sha3 0.9.1", - "solana-program", + "solana-program 1.18.22", "solana-sdk", "subtle", - "thiserror", + "thiserror 1.0.69", "zeroize", ] @@ -6428,7 +7502,7 @@ dependencies = [ "rand 0.8.5", "rustc-demangle", "scroll", - "thiserror", + "thiserror 1.0.69", "winapi", ] @@ -6462,7 +7536,7 @@ checksum = "2785042005954aec5d5db7fcb99a78754b222be906a89d10a3d66ebdbc8e9548" dependencies = [ "anchor-lang", "bytemuck", - "solana-program", + "solana-program 2.1.8", "spl-concurrent-merkle-tree", "spl-noop", ] @@ -6474,13 +7548,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f" dependencies = [ "assert_matches", - "borsh 0.10.3", - "num-derive 0.4.1", + "borsh 0.10.4", + "num-derive 0.4.2", "num-traits", - "solana-program", + "solana-program 1.18.22", "spl-token", "spl-token-2022", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -6490,8 +7564,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a14033366e14117679851c7759c3d66c6430a495f0523bd88076d3a275828931" dependencies = [ "bytemuck", - "solana-program", - "thiserror", + "solana-program 1.18.22", + "thiserror 1.0.69", ] [[package]] @@ -6501,32 +7575,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f" dependencies = [ "bytemuck", - "solana-program", + "solana-program 1.18.22", "spl-discriminator-derive", ] [[package]] name = "spl-discriminator-derive" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b" +checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93" dependencies = [ "quote", "spl-discriminator-syn", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "spl-discriminator-syn" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2" +checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.66", - "thiserror", + "syn 2.0.96", + "thiserror 1.0.69", ] [[package]] @@ -6535,7 +7609,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f180b03318c3dbab3ef4e1e4d46d5211ae3c780940dd0a28695aba4b59a75a" dependencies = [ - "solana-program", + "solana-program 1.18.22", ] [[package]] @@ -6544,7 +7618,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd67ea3d0070a12ff141f5da46f9695f49384a03bce1203a5608f5739437950" dependencies = [ - "solana-program", + "solana-program 1.18.22", ] [[package]] @@ -6554,10 +7628,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" dependencies = [ "base64 0.21.7", - "borsh 0.10.3", + "borsh 0.10.4", "bytemuck", "serde", - "solana-program", + "solana-program 1.18.22", "solana-zk-token-sdk", "spl-program-error", ] @@ -6568,33 +7642,33 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c" dependencies = [ - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", - "solana-program", + "solana-program 1.18.22", "spl-program-error-derive", - "thiserror", + "thiserror 1.0.69", ] [[package]] name = "spl-program-error-derive" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c" +checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474" dependencies = [ "proc-macro2", "quote", "sha2 0.10.8", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "spl-tlv-account-resolution" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f335787add7fa711819f9e7c573f8145a5358a709446fe2d24bf2a88117c90" +checksum = "615d381f48ddd2bb3c57c7f7fb207591a2a05054639b18a62e785117dd7a8683" dependencies = [ "bytemuck", - "solana-program", + "solana-program 1.18.22", "spl-discriminator", "spl-pod", "spl-program-error", @@ -6612,8 +7686,8 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "num_enum 0.6.1", - "solana-program", - "thiserror", + "solana-program 1.18.22", + "thiserror 1.0.69", ] [[package]] @@ -6624,10 +7698,10 @@ checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059" dependencies = [ "arrayref", "bytemuck", - "num-derive 0.4.1", + "num-derive 0.4.2", "num-traits", - "num_enum 0.7.2", - "solana-program", + "num_enum 0.7.3", + "solana-program 1.18.22", "solana-security-txt", "solana-zk-token-sdk", "spl-memo", @@ -6637,7 +7711,7 @@ dependencies = [ "spl-token-metadata-interface", "spl-transfer-hook-interface", "spl-type-length-value", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -6647,7 +7721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b889509d49fa74a4a033ca5dae6c2307e9e918122d97e58562f5c4ffa795c75d" dependencies = [ "bytemuck", - "solana-program", + "solana-program 1.18.22", "spl-discriminator", "spl-pod", "spl-program-error", @@ -6659,8 +7733,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c16ce3ba6979645fb7627aa1e435576172dd63088dc7848cb09aa331fa1fe4f" dependencies = [ - "borsh 0.10.3", - "solana-program", + "borsh 0.10.4", + "solana-program 1.18.22", "spl-discriminator", "spl-pod", "spl-program-error", @@ -6675,7 +7749,7 @@ checksum = "7aabdb7c471566f6ddcee724beb8618449ea24b399e58d464d6b5bc7db550259" dependencies = [ "arrayref", "bytemuck", - "solana-program", + "solana-program 1.18.22", "spl-discriminator", "spl-pod", "spl-program-error", @@ -6690,7 +7764,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac" dependencies = [ "bytemuck", - "solana-program", + "solana-program 1.18.22", "spl-discriminator", "spl-pod", "spl-program-error", @@ -6698,11 +7772,10 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" dependencies = [ - "itertools 0.11.0", "nom", "unicode_categories", ] @@ -6723,7 +7796,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "atoi", "base64 0.13.1", "bitflags 1.3.2", @@ -6750,7 +7823,7 @@ dependencies = [ "log", "md-5", "memchr", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "once_cell", "paste", "percent-encoding", @@ -6766,8 +7839,8 @@ dependencies = [ "sqlformat", "sqlx-rt", "stringprep", - "thiserror", - "time 0.3.29", + "thiserror 1.0.69", + "time 0.3.37", "tokio-stream", "url", "uuid", @@ -6826,16 +7899,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70a313e115c2cd9a88d99d60386bc88641c853d468b2c3bc454c294f385fc084" dependencies = [ - "async-channel 2.1.0", - "async-io 2.3.2", + "async-channel 2.3.1", + "async-io", "atomic", "crossbeam-channel", "futures", - "getrandom 0.2.10", - "parking_lot 0.12.1", + "getrandom 0.2.15", + "parking_lot 0.12.3", "rand 0.8.5", "seahash", - "thiserror", + "thiserror 1.0.69", "tracing", "wg", "xxhash-rust", @@ -6843,13 +7916,13 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -6864,6 +7937,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.4.1" @@ -6883,27 +7962,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "syn_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "sync_wrapper" version = "0.1.2" @@ -6922,6 +7989,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -6951,22 +8029,23 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", - "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.18", - "windows-sys 0.48.0", + "fastrand", + "getrandom 0.2.15", + "once_cell", + "rustix", + "windows-sys 0.59.0", ] [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -6977,45 +8056,87 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl 2.0.11", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 2.10.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.45" @@ -7029,12 +8150,16 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -7048,10 +8173,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ + "num-conv", "time-core", ] @@ -7068,17 +8194,27 @@ dependencies = [ "rand 0.7.3", "rustc-hash", "sha2 0.9.9", - "thiserror", + "thiserror 1.0.69", "unicode-normalization", "wasm-bindgen", "zeroize", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -7091,33 +8227,42 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] @@ -7153,9 +8298,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -7174,14 +8319,14 @@ dependencies = [ "tokio", "tokio-rustls 0.24.1", "tungstenite", - "webpki-roots 0.25.2", + "webpki-roots 0.25.4", ] [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -7189,7 +8334,6 @@ dependencies = [ "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -7203,14 +8347,26 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit 0.19.15", +] + +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.22.22", ] [[package]] @@ -7228,33 +8384,82 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", + "serde", + "serde_spanned", "toml_datetime", - "winnow 0.5.16", + "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.5.16", + "winnow 0.6.24", ] [[package]] -name = "toml_edit" -version = "0.22.22" +name = "tonic" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ - "indexmap 2.6.0", - "toml_datetime", - "winnow 0.6.20", + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls 0.21.12", + "rustls-native-certs", + "rustls-pemfile", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "tonic-health" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" +dependencies = [ + "async-stream", + "prost", + "tokio", + "tokio-stream", + "tonic", ] [[package]] @@ -7263,6 +8468,15 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -7288,23 +8502,22 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -7313,20 +8526,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -7343,11 +8556,29 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time 0.2.4", +] + [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", @@ -7355,9 +8586,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -7381,9 +8612,9 @@ dependencies = [ "anchor-client", "anyhow", "bs58 0.4.0", - "clap 4.4.8", + "clap 4.5.26", "digital_asset_types", - "env_logger 0.10.0", + "env_logger 0.10.2", "flatbuffers", "futures", "hex", @@ -7398,16 +8629,16 @@ dependencies = [ "spl-account-compression", "spl-noop", "sqlx", - "thiserror", + "thiserror 1.0.69", "tokio", "txn_forwarder", ] [[package]] name = "triomphe" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" +checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" dependencies = [ "serde", "stable_deref_trait", @@ -7415,9 +8646,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -7434,7 +8665,7 @@ dependencies = [ "rand 0.8.5", "rustls 0.21.12", "sha1", - "thiserror", + "thiserror 1.0.69", "url", "utf-8", "webpki-roots 0.24.0", @@ -7445,8 +8676,8 @@ name = "txn_forwarder" version = "0.7.12" dependencies = [ "anyhow", - "clap 4.4.8", - "env_logger 0.10.0", + "clap 4.5.26", + "env_logger 0.10.2", "figment", "flatbuffers", "futures", @@ -7460,7 +8691,7 @@ dependencies = [ "solana-client", "solana-sdk", "solana-transaction-status", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", ] @@ -7471,62 +8702,77 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uncased" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" dependencies = [ "version_check", ] [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unicode_categories" @@ -7555,9 +8801,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -7583,34 +8829,52 @@ dependencies = [ [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.4.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.15", "serde", ] @@ -7622,9 +8886,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "3ef4c4aa54d5d05a279399bfa921ec387b7aba77caf7a682ae8d86785b8fdad2" [[package]] name = "vcpkg" @@ -7638,11 +8902,23 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "rustc_version", + "rustversion", + "time 0.3.37", +] + [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "void" @@ -7650,12 +8926,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - [[package]] name = "want" version = "0.3.1" @@ -7689,49 +8959,56 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7739,28 +9016,51 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -7772,7 +9072,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.2", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7796,30 +9096,31 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "wg" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dadf90865f15d5c2d87f126a56ce3715b3a233641acdd10f59200aa7f4c81fb9" +checksum = "7aafc5e81e847f05d6770e074faf7b1cd4a5dec9a0e88eac5d55e20fdfebee9a" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.4.0", "futures-core", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project-lite", "triomphe", ] [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.5.8", + "wasite", "web-sys", ] @@ -7841,11 +9142,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -7855,12 +9156,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -7878,7 +9179,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -7898,18 +9208,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -7920,9 +9230,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -7932,9 +9242,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -7944,15 +9254,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -7962,9 +9272,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -7974,9 +9284,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -7986,9 +9296,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -7998,24 +9308,24 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.5.16" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] @@ -8030,6 +9340,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -8053,58 +9375,155 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror", - "time 0.3.29", + "thiserror 1.0.69", + "time 0.3.37", ] [[package]] name = "xxhash-rust" -version = "0.8.7" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yasna" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "linked-hash-map", + "time 0.3.37", ] [[package]] -name = "yansi" -version = "1.0.0-rc.1" +name = "yellowstone-grpc-client" +version = "1.15.3+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" +dependencies = [ + "bytes", + "futures", + "thiserror 1.0.69", + "tonic", + "tonic-health", + "yellowstone-grpc-proto", +] + +[[package]] +name = "yellowstone-grpc-proto" +version = "1.14.2+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" +dependencies = [ + "anyhow", + "bincode", + "prost", + "protobuf-src", + "solana-account-decoder", + "solana-sdk", + "solana-transaction-status", + "tonic", + "tonic-build", +] + +[[package]] +name = "yellowstone-grpc-tools" +version = "1.0.0-rc.11+solana.1.18.22" +source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.15.1+solana.1.18.22#0bdedb5841d2eea663b8f2f441d37fea83e65933" +dependencies = [ + "anyhow", + "async-trait", + "atty", + "cargo-lock", + "clap 4.5.26", + "futures", + "git-version", + "hyper", + "json5", + "lazy_static", + "project-root", + "prometheus", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "tokio-stream", + "tonic", + "tonic-health", + "tracing", + "tracing-subscriber", + "vergen", + "yellowstone-grpc-client", + "yellowstone-grpc-proto", +] + +[[package]] +name = "yoke" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] [[package]] -name = "yasna" -version = "0.5.2" +name = "yoke-derive" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ - "time 0.3.29", + "proc-macro2", + "quote", + "syn 2.0.96", + "synstructure 0.13.1", ] [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", + "synstructure 0.13.1", ] [[package]] @@ -8124,7 +9543,29 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.96", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", ] [[package]] @@ -8148,11 +9589,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 101452cee..43e43d397 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "bubblegum", "blockbuster", "core", "das_api", @@ -9,6 +10,7 @@ members = [ "migration", "nft_ingester", "ops", + "grpc-ingest", "program_transformers", "tools/acc_forwarder", "tools/bgtask_creator", @@ -26,24 +28,30 @@ repository = "https://github.com/metaplex-foundation/digital-asset-rpc-infrastru version = "0.7.2" [workspace.dependencies] + anchor-client = "0.29.0" anchor-lang = "0.29.0" anyhow = "1.0.75" async-std = "1.0.0" +async-stream = "0.3.5" async-trait = "0.1.60" +atty = "0.2.14" backon = "0.4.1" -blockbuster = {path = "blockbuster"} +blockbuster = { path = "blockbuster" } borsh = "~0.10.3" borsh-derive = "~0.10.3" bs58 = "0.4.0" -bytemuck = {version = "1.14.0", features = ["derive"]} +bytemuck = { version = "1.14.0", features = ["derive"] } cadence = "0.29.0" cadence-macros = "0.29.0" +cargo-lock = "9.0.0" chrono = "0.4.19" clap = "4.2.2" -das-core = {path = "core"} -das_api = {path = "das_api"} -digital_asset_types = {path = "digital_asset_types"} +das-core = { path = "core" } +das-bubblegum = { path = "bubblegum" } +das_api = { path = "das_api" } +derive_more = { version = "0.99.17" } +digital_asset_types = { path = "digital_asset_types" } enum-iterator = "1.2.0" enum-iterator-derive = "1.1.0" env_logger = "0.10.0" @@ -52,6 +60,7 @@ figment = "0.10.8" flatbuffers = "23.1.21" function_name = "0.3.0" futures = "0.3.28" +git-version = "0.3.5" heck = "0.5.0" hex = "0.4.3" hyper = "0.14.23" @@ -59,31 +68,37 @@ indexmap = "1.9.3" indicatif = "0.17.5" insta = "1.34.0" itertools = "0.10.1" +json5 = "0.4.1" jsonpath_lib = "0.3.0" jsonrpsee = "0.16.2" jsonrpsee-core = "0.16.2" lazy_static = "1.4.0" log = "0.4.17" +lru = "0.12.3" metrics = "0.20.1" -migration = {path = "migration"} +migration = { path = "migration" } mime_guess = "2.0.4" -mpl-bubblegum = "1.2.0" mpl-account-compression = "0.4.2" -mpl-core = {version = "0.8.0-beta.1", features = ["serde"]} +mpl-bubblegum = "1.4.0" +mpl-core = { version = "0.8.0-beta.1", features = ["serde"] } mpl-noop = "0.2.1" mpl-token-metadata = "4.1.1" -nft_ingester = {path = "nft_ingester"} +nft_ingester = { path = "nft_ingester" } num-derive = "0.3.3" num-traits = "0.2.15" once_cell = "1.19.0" open-rpc-derive = "0.0.4" open-rpc-schema = "0.0.4" +opentelemetry = "0.21.0" +opentelemetry-jaeger = "0.20.0" +opentelemetry_sdk = "0.21.1" plerkle_messenger = "1.6.0" plerkle_serialization = "1.8.0" -program_transformers = {path = "program_transformers"} +program_transformers = { path = "program_transformers" } prometheus = "0.13.3" proxy-wasm = "0.2.0" rand = "0.8.5" +redis = "0.25.3" regex = "1.6.0" reqwest = "0.11.13" rust-crypto = "0.2.36" @@ -94,6 +109,7 @@ sea-orm-migration = "0.10.6" sea-query = "0.28.1" serde = "1.0.137" serde_json = "1.0.81" +serde_yaml = "0.9.34" serial_test = "2.0.0" solana-account-decoder = "~1.18.15" solana-client = "~1.18.15" @@ -106,11 +122,12 @@ spl-account-compression = "0.4.2" spl-associated-token-account = ">= 1.1.3, < 3.0" spl-concurrent-merkle-tree = "0.4.1" spl-noop = "0.2.0" -spl-pod = {version = "0.1.0", features = ["serde-traits"]} +spl-pod = { version = "0.1.0", features = ["serde-traits"] } spl-token = ">= 3.5.0, < 5.0" -spl-token-2022 = {version = "1.0", features = ["no-entrypoint"]} +spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] } spl-token-group-interface = "0.1.0" spl-token-metadata-interface = "0.2.0" +sha3 = "0.10.8" sqlx = "0.6.2" stretto = "0.8.4" thiserror = "1.0.31" @@ -119,11 +136,16 @@ tokio-stream = "0.1.14" tower = "0.4.13" tower-http = "0.3.5" tracing = "0.1.35" +tracing-opentelemetry = "0.22.0" tracing-subscriber = "0.3.16" -txn_forwarder = {path = "tools/txn_forwarder"} +txn_forwarder = { path = "tools/txn_forwarder" } url = "2.3.1" +vergen = "8.2.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" +yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22" } # tag is geyser plugin +yellowstone-grpc-tools = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.15.1+solana.1.18.22", default-features = false } # tag is geyser plugin [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/Ingest.Dockerfile b/Ingest.Dockerfile index a0a6023d6..8ff559bfe 100644 --- a/Ingest.Dockerfile +++ b/Ingest.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Load.Dockerfile b/Load.Dockerfile index ae4ee3ec1..91e754c4b 100644 --- a/Load.Dockerfile +++ b/Load.Dockerfile @@ -1,5 +1,5 @@ FROM das-api/builder AS files -FROM rust:1.75-slim-bullseye +FROM rust:1.79-slim-bullseye ARG APP=/usr/src/app RUN apt update \ && apt install -y curl ca-certificates tzdata \ diff --git a/Migrator.Dockerfile b/Migrator.Dockerfile index 63c72bc3d..e21f7d77e 100644 --- a/Migrator.Dockerfile +++ b/Migrator.Dockerfile @@ -1,6 +1,6 @@ FROM das-api/builder AS files -FROM rust:1.76-bullseye +FROM rust:1.79-bullseye COPY init.sql /init.sql ENV INIT_FILE_PATH=/init.sql COPY --from=files /das/migration /bins/migration diff --git a/Proxy.Dockerfile b/Proxy.Dockerfile index 52e35de55..cfee4fb6d 100644 --- a/Proxy.Dockerfile +++ b/Proxy.Dockerfile @@ -1,9 +1,10 @@ -FROM rust:1.76-bullseye AS builder +FROM rust:1.79-bullseye AS builder RUN cargo install wasm-pack RUN mkdir /rust COPY ./Cargo.toml /rust COPY ./core /rust/core +COPY ./backfill /rust/backfill COPY ./das_api /rust/das_api COPY ./digital_asset_types /rust/digital_asset_types COPY ./integration_tests /rust/integration_tests diff --git a/README.md b/README.md index 87d87aa79..ce7af8e5e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,17 @@ super-fast querying and searching, as well as serves the merkle proofs needed to The API specification is located here https://github.com/metaplex-foundation/api-specifications This spec is what providers of this api must implement against. +#### GRPC-INGEST [/grpc-ingest/README.md](/grpc-ingest/README.md) + +This repo also houses the GRPC-INGEST component. This is a reimplementation of nft-ingester +This components separates grpc and ingester into two separate components. This is to allow for more flexibility in the future also increases the performance of the system. + +The two components are: + +- grpc2redis - This component listens to a gRPC stream (triton's Dragon's Mouth gRPC) for account and transaction updates and pushes them to a pipeline that flushes data to redis at regular intervals in real time. + +- ingester - This component listens to the redis stream and processes the data using program transformers. It also downloads token metadata json and stores them in a postgres db using sea-orm. + ### Infrastructure and Deployment Examples Along with the above rust binaries, this repo also maintains examples and best practice settings for running the entire infrastructure. diff --git a/blockbuster/src/instruction.rs b/blockbuster/src/instruction.rs index 4345e29ed..6a5a04f54 100644 --- a/blockbuster/src/instruction.rs +++ b/blockbuster/src/instruction.rs @@ -14,7 +14,7 @@ pub struct InstructionBundle<'a> { pub slot: u64, } -impl<'a> Default for InstructionBundle<'a> { +impl Default for InstructionBundle<'_> { fn default() -> Self { InstructionBundle { txn_id: "", diff --git a/blockbuster/src/lib.rs b/blockbuster/src/lib.rs index 983c72897..e8e6eeedf 100644 --- a/blockbuster/src/lib.rs +++ b/blockbuster/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(deprecated)] pub mod error; pub mod instruction; pub mod program_handler; diff --git a/bubblegum/Cargo.toml b/bubblegum/Cargo.toml new file mode 100644 index 000000000..0ecd4704a --- /dev/null +++ b/bubblegum/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "das-bubblegum" +version = { workspace = true } +edition = { workspace = true } +repository = { workspace = true } +publish = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +blockbuster = { workspace = true } +bs58 = { workspace = true } +das-core = { workspace = true } +solana-client = { workspace = true } +borsh = { workspace = true } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } +anchor-client = { workspace = true } +futures = { workspace = true } +clap = { workspace = true } +log = { workspace = true } +solana-program = { workspace = true } +program_transformers = { workspace = true } +heck = { workspace = true } +mpl-bubblegum = { workspace = true } +num-traits = { workspace = true } +sea-orm = { workspace = true } +serde_json = { workspace = true } +solana-sdk = { workspace = true } +sha3 = { workspace = true } +solana-transaction-status = { workspace = true } +spl-account-compression = { workspace = true, features = ["no-entrypoint"] } +spl-token = { workspace = true, features = ["no-entrypoint"] } +sqlx = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["time"] } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/bubblegum/README.md b/bubblegum/README.md new file mode 100644 index 000000000..e98983303 --- /dev/null +++ b/bubblegum/README.md @@ -0,0 +1,24 @@ +## DAS Backfill + +The DAS Backfill library facilitates the initial setup and data backfilling for DAS, focusing on the bubblegum program. This program's indexing heavily relies on transaction data. While the library supports parallel backfilling across different trees, it ensures that transactions within each tree are processed sequentially. This approach guarantees accurate representation of every modification in the merkle tree within DAS. + +## Usage + +```rust +use das_backfill::{ + BubblegumBackfillArgs, + BubblegumBackfillContext, + start_bubblegum_backfill +}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let database_pool = sqlx::PgPool::connect("your_database_url").await?; + let solana_rpc = Rpc::new("your_solana_rpc_url"); + + let context = BubblegumBackfillContext::new(database_pool, solana_rpc); + let args = BubblegumBackfillArgs::parse(); // Parses args from CLI + + start_bubblegum_backfill(context, args).await +} +``` diff --git a/bubblegum/src/backfill/gap.rs b/bubblegum/src/backfill/gap.rs new file mode 100644 index 000000000..e167e888d --- /dev/null +++ b/bubblegum/src/backfill/gap.rs @@ -0,0 +1,139 @@ +use crate::{error::ErrorKind, Rpc}; +use anyhow::Result; +use clap::Args; +use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; +use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; +use solana_sdk::{pubkey::Pubkey, signature::Signature}; +use std::str::FromStr; +use tokio::sync::mpsc::Sender; + +const GET_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1000; + +#[derive(Debug, Clone, Args)] +pub struct ConfigBackfiller { + /// Solana RPC URL + #[arg(long, env)] + pub solana_rpc_url: String, +} + +const TREE_GAP_SQL: &str = r#" +WITH sequenced_data AS ( + SELECT + tree, + seq, + LEAD(seq) OVER (ORDER BY seq ASC) AS next_seq, + tx AS current_tx, + LEAD(tx) OVER (ORDER BY seq ASC) AS next_tx + FROM + cl_audits_v2 + WHERE + tree = $1 +), +gaps AS ( + SELECT + tree, + seq AS gap_start_seq, + next_seq AS gap_end_seq, + current_tx AS lower_bound_tx, + next_tx AS upper_bound_tx + FROM + sequenced_data + WHERE + next_seq IS NOT NULL AND + next_seq - seq > 1 +) +SELECT + tree, + gap_start_seq, + gap_end_seq, + lower_bound_tx, + upper_bound_tx +FROM + gaps +ORDER BY + gap_start_seq; +"#; + +#[derive(Debug, FromQueryResult, PartialEq, Clone)] +pub struct TreeGapModel { + pub tree: Vec, + pub gap_start_seq: i64, + pub gap_end_seq: i64, + pub lower_bound_tx: Vec, + pub upper_bound_tx: Vec, +} + +impl TreeGapModel { + pub async fn find(conn: &DatabaseConnection, tree: Pubkey) -> Result, ErrorKind> { + let statement = Statement::from_sql_and_values( + DbBackend::Postgres, + TREE_GAP_SQL, + vec![Value::Bytes(Some(Box::new(tree.as_ref().to_vec())))], + ); + + TreeGapModel::find_by_statement(statement) + .all(conn) + .await + .map_err(Into::into) + } +} + +impl TryFrom for TreeGapFill { + type Error = ErrorKind; + + fn try_from(model: TreeGapModel) -> Result { + let tree = Pubkey::try_from(model.tree).map_err(|_| ErrorKind::TryFromPubkey)?; + let upper = + Signature::try_from(model.upper_bound_tx).map_err(|_| ErrorKind::TryFromSignature)?; + let lower = + Signature::try_from(model.lower_bound_tx).map_err(|_| ErrorKind::TryFromSignature)?; + + Ok(Self::new(tree, Some(upper), Some(lower))) + } +} + +pub struct TreeGapFill { + tree: Pubkey, + before: Option, + until: Option, +} + +impl TreeGapFill { + pub const fn new(tree: Pubkey, before: Option, until: Option) -> Self { + Self { + tree, + before, + until, + } + } + + pub async fn crawl(&self, client: Rpc, sender: Sender) -> Result<()> { + let mut before = self.before; + + loop { + let sigs = client + .get_signatures_for_address(&self.tree, before, self.until) + .await?; + let sig_count = sigs.len(); + + let successful_transactions = sigs + .into_iter() + .filter(|transaction| transaction.err.is_none()) + .collect::>(); + + for sig in successful_transactions.iter() { + let sig = Signature::from_str(&sig.signature)?; + + sender.send(sig).await?; + + before = Some(sig); + } + + if sig_count < GET_SIGNATURES_FOR_ADDRESS_LIMIT { + break; + } + } + + Ok(()) + } +} diff --git a/bubblegum/src/backfill/mod.rs b/bubblegum/src/backfill/mod.rs new file mode 100644 index 000000000..5a4a874d6 --- /dev/null +++ b/bubblegum/src/backfill/mod.rs @@ -0,0 +1,2 @@ +pub mod gap; +pub mod worker; diff --git a/bubblegum/src/backfill/worker/gap.rs b/bubblegum/src/backfill/worker/gap.rs new file mode 100644 index 000000000..68523fdbe --- /dev/null +++ b/bubblegum/src/backfill/worker/gap.rs @@ -0,0 +1,64 @@ +use anyhow::Result; +use clap::Parser; +use das_core::Rpc; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use solana_sdk::signature::Signature; +use tokio::{ + sync::mpsc::{channel, Sender}, + task::JoinHandle, +}; + +use crate::{backfill::gap::TreeGapFill, BubblegumContext}; + +#[derive(Parser, Debug, Clone)] +pub struct GapWorkerArgs { + /// The size of the signature channel. + #[arg(long, env, default_value = "1000")] + pub gap_channel_size: usize, + + /// The number of gap workers. + #[arg(long, env, default_value = "25")] + pub gap_worker_count: usize, +} + +impl GapWorkerArgs { + pub fn start( + &self, + context: BubblegumContext, + forward: Sender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (gap_sender, mut gap_receiver) = channel::(self.gap_channel_size); + let gap_worker_count = self.gap_worker_count; + + let handler = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + let sender = forward.clone(); + + while let Some(gap) = gap_receiver.recv().await { + if handlers.len() >= gap_worker_count { + handlers.next().await; + } + + let client = context.solana_rpc.clone(); + let sender = sender.clone(); + + let handle = spawn_crawl_worker(client, sender, gap); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + }); + + Ok((handler, gap_sender)) + } +} + +fn spawn_crawl_worker(client: Rpc, sender: Sender, gap: TreeGapFill) -> JoinHandle<()> { + tokio::spawn(async move { + if let Err(e) = gap.crawl(client, sender).await { + error!("tree transaction: {:?}", e); + } + }) +} diff --git a/bubblegum/src/backfill/worker/mod.rs b/bubblegum/src/backfill/worker/mod.rs new file mode 100644 index 000000000..0bc7f9e36 --- /dev/null +++ b/bubblegum/src/backfill/worker/mod.rs @@ -0,0 +1,9 @@ +mod gap; +mod program_transformer; +mod transaction; +mod tree; + +pub use gap::GapWorkerArgs; +pub use program_transformer::ProgramTransformerWorkerArgs; +pub use transaction::SignatureWorkerArgs; +pub use tree::TreeWorkerArgs; diff --git a/bubblegum/src/backfill/worker/program_transformer.rs b/bubblegum/src/backfill/worker/program_transformer.rs new file mode 100644 index 000000000..75f75c0a7 --- /dev/null +++ b/bubblegum/src/backfill/worker/program_transformer.rs @@ -0,0 +1,70 @@ +use anyhow::Result; +use clap::Parser; +use das_core::{create_download_metadata_notifier, DownloadMetadataInfo}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use log::error; +use program_transformers::{ProgramTransformer, TransactionInfo}; +use std::sync::Arc; +use tokio::sync::mpsc::{channel, Sender, UnboundedSender}; +use tokio::task::JoinHandle; + +use crate::BubblegumContext; + +#[derive(Parser, Debug, Clone)] +pub struct ProgramTransformerWorkerArgs { + #[arg(long, env, default_value = "100000")] + pub program_transformer_channel_size: usize, + #[arg(long, env, default_value = "50")] + pub program_transformer_worker_count: usize, +} + +impl ProgramTransformerWorkerArgs { + pub fn start( + &self, + context: BubblegumContext, + forwarder: UnboundedSender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (sender, mut receiver) = + channel::(self.program_transformer_channel_size); + + let worker_forwarder = forwarder.clone(); + let worker_pool = context.database_pool.clone(); + let worker_count = self.program_transformer_worker_count; + let handle = tokio::spawn(async move { + let download_metadata_notifier = + create_download_metadata_notifier(worker_forwarder.clone()).await; + let program_transformer = Arc::new(ProgramTransformer::new( + worker_pool.clone(), + download_metadata_notifier, + )); + + let mut handlers = FuturesUnordered::new(); + + while let Some(transaction) = receiver.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } + + let program_transformer_clone = Arc::clone(&program_transformer); + let handle = tokio::spawn(async move { + if let Err(err) = program_transformer_clone + .handle_transaction(&transaction) + .await + { + error!( + "Failed to handle bubblegum instruction for txn {:?}: {:?}", + transaction.signature, err + ); + } + }); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + }); + + Ok((handle, sender)) + } +} diff --git a/bubblegum/src/backfill/worker/transaction.rs b/bubblegum/src/backfill/worker/transaction.rs new file mode 100644 index 000000000..910b79a92 --- /dev/null +++ b/bubblegum/src/backfill/worker/transaction.rs @@ -0,0 +1,205 @@ +use crate::{error::ErrorKind, BubblegumContext}; +use anyhow::Result; +use clap::Parser; +use das_core::Rpc; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use program_transformers::TransactionInfo; +use solana_program::pubkey::Pubkey; +use solana_sdk::instruction::CompiledInstruction; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::VersionedTransaction; +use solana_transaction_status::{ + option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta, + InnerInstruction, InnerInstructions, UiInstruction, +}; +use tokio::{ + sync::mpsc::{channel, Sender}, + task::JoinHandle, +}; + +pub struct PubkeyString(pub String); + +impl TryFrom for Pubkey { + type Error = ErrorKind; + + fn try_from(value: PubkeyString) -> Result { + let decoded_bytes = bs58::decode(value.0) + .into_vec() + .map_err(|e| ErrorKind::Generic(e.to_string()))?; + + Pubkey::try_from(decoded_bytes) + .map_err(|_| ErrorKind::Generic("unable to convert pubkey".to_string())) + } +} + +#[derive(Debug)] +pub struct FetchedEncodedTransactionWithStatusMeta(pub EncodedConfirmedTransactionWithStatusMeta); + +impl TryFrom for TransactionInfo { + type Error = ErrorKind; + + fn try_from( + fetched_transaction: FetchedEncodedTransactionWithStatusMeta, + ) -> Result { + let mut account_keys = Vec::new(); + let encoded_transaction_with_status_meta = fetched_transaction.0; + + let ui_transaction: VersionedTransaction = encoded_transaction_with_status_meta + .transaction + .transaction + .decode() + .ok_or(ErrorKind::Generic( + "unable to decode transaction".to_string(), + ))?; + + let signature = ui_transaction.signatures[0]; + + let msg = ui_transaction.message; + + let meta = encoded_transaction_with_status_meta + .transaction + .meta + .ok_or(ErrorKind::Generic( + "transaction metadata is missing".to_string(), + ))?; + + for address in msg.static_account_keys().iter().copied() { + account_keys.push(address); + } + + let ui_loaded_addresses = match meta.loaded_addresses { + OptionSerializer::Some(addresses) => addresses, + OptionSerializer::None => { + return Err(ErrorKind::Generic( + "loaded addresses data is missing".to_string(), + )) + } + OptionSerializer::Skip => { + return Err(ErrorKind::Generic( + "loaded addresses are skipped".to_string(), + )); + } + }; + + let writtable_loaded_addresses = ui_loaded_addresses.writable; + let readable_loaded_addresses = ui_loaded_addresses.readonly; + + if msg.address_table_lookups().is_some() { + for address in writtable_loaded_addresses { + account_keys.push(PubkeyString(address).try_into()?); + } + + for address in readable_loaded_addresses { + account_keys.push(PubkeyString(address).try_into()?); + } + } + + let mut meta_inner_instructions = Vec::new(); + + if let OptionSerializer::Some(inner_instructions) = meta.inner_instructions { + for ix in inner_instructions { + let mut instructions = Vec::new(); + + for inner in ix.instructions { + if let UiInstruction::Compiled(compiled) = inner { + instructions.push(InnerInstruction { + stack_height: compiled.stack_height, + instruction: CompiledInstruction { + program_id_index: compiled.program_id_index, + accounts: compiled.accounts, + data: bs58::decode(compiled.data).into_vec().map_err(|e| { + ErrorKind::Generic(format!("Error decoding data: {}", e)) + })?, + }, + }); + } + } + + meta_inner_instructions.push(InnerInstructions { + index: ix.index, + instructions, + }); + } + } + + Ok(Self { + slot: encoded_transaction_with_status_meta.slot, + account_keys, + signature, + message_instructions: msg.instructions().to_vec(), + meta_inner_instructions, + }) + } +} + +#[derive(Parser, Clone, Debug)] +pub struct SignatureWorkerArgs { + /// The size of the signature channel. + #[arg(long, env, default_value = "100000")] + pub signature_channel_size: usize, + /// The number of transaction workers. + #[arg(long, env, default_value = "50")] + pub signature_worker_count: usize, +} + +type TransactionSender = Sender; + +impl SignatureWorkerArgs { + pub fn start( + &self, + context: BubblegumContext, + forwarder: TransactionSender, + ) -> Result<(JoinHandle<()>, Sender)> { + let (sig_sender, mut sig_receiver) = channel::(self.signature_channel_size); + let worker_count = self.signature_worker_count; + + let handle = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + + while let Some(signature) = sig_receiver.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } + + let solana_rpc = context.solana_rpc.clone(); + let transaction_sender = forwarder.clone(); + + let handle = spawn_transaction_worker(solana_rpc, transaction_sender, signature); + + handlers.push(handle); + } + + futures::future::join_all(handlers).await; + }); + + Ok((handle, sig_sender)) + } +} + +async fn queue_transaction<'a>( + client: Rpc, + sender: Sender, + signature: Signature, +) -> Result<(), ErrorKind> { + let transaction = client.get_transaction(&signature).await?; + + sender + .send(FetchedEncodedTransactionWithStatusMeta(transaction).try_into()?) + .await + .map_err(|e| ErrorKind::Generic(e.to_string()))?; + + Ok(()) +} + +fn spawn_transaction_worker( + client: Rpc, + sender: Sender, + signature: Signature, +) -> JoinHandle<()> { + tokio::spawn(async move { + if let Err(e) = queue_transaction(client, sender, signature).await { + error!("queue transaction: {:?}", e); + } + }) +} diff --git a/bubblegum/src/backfill/worker/tree.rs b/bubblegum/src/backfill/worker/tree.rs new file mode 100644 index 000000000..761b674da --- /dev/null +++ b/bubblegum/src/backfill/worker/tree.rs @@ -0,0 +1,128 @@ +use std::sync::Arc; + +use crate::{ + backfill::gap::{TreeGapFill, TreeGapModel}, + tree::TreeResponse, + BubblegumContext, +}; +use anyhow::Result; +use clap::Parser; +use das_core::{DownloadMetadataJsonRetryConfig, MetadataJsonDownloadWorkerArgs}; +use digital_asset_types::dao::cl_audits_v2; +use log::error; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector}; +use solana_sdk::signature::Signature; +use tokio::task::JoinHandle; + +use super::{GapWorkerArgs, ProgramTransformerWorkerArgs, SignatureWorkerArgs}; + +#[derive(Debug, Clone, Parser)] +pub struct TreeWorkerArgs { + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + + #[clap(flatten)] + pub signature_worker: SignatureWorkerArgs, + + #[clap(flatten)] + pub gap_worker: GapWorkerArgs, + + #[clap(flatten)] + pub program_transformer_worker: ProgramTransformerWorkerArgs, + + #[clap(long, env, default_value = "false")] + pub force: bool, +} +impl TreeWorkerArgs { + pub fn start(&self, context: BubblegumContext, tree: TreeResponse) -> JoinHandle> { + let db_pool = context.database_pool.clone(); + let metadata_json_download_db_pool = context.database_pool.clone(); + + let program_transformer_context = context.clone(); + let signature_context = context.clone(); + + let metadata_json_download_worker_args = self.metadata_json_download_worker.clone(); + let program_transformer_worker_args = self.program_transformer_worker.clone(); + let signature_worker_args = self.signature_worker.clone(); + let gap_worker_args = self.gap_worker.clone(); + let force = self.force; + let download_config = Arc::new(DownloadMetadataJsonRetryConfig::default()); + + tokio::spawn(async move { + let (metadata_json_download_worker, metadata_json_download_sender) = + metadata_json_download_worker_args + .start(metadata_json_download_db_pool, download_config)?; + + let (program_transformer_worker, transaction_info_sender) = + program_transformer_worker_args + .start(program_transformer_context, metadata_json_download_sender)?; + + let (signature_worker, signature_sender) = + signature_worker_args.start(signature_context, transaction_info_sender)?; + + let (gap_worker, tree_gap_sender) = gap_worker_args.start(context, signature_sender)?; + + { + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(db_pool); + + let mut gaps = TreeGapModel::find(&conn, tree.pubkey) + .await? + .into_iter() + .map(TryInto::try_into) + .collect::, _>>()?; + + let upper_known_seq = if force { + None + } else { + cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_desc(cl_audits_v2::Column::Seq) + .one(&conn) + .await? + }; + + let lower_known_seq = if force { + None + } else { + cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) + .order_by_asc(cl_audits_v2::Column::Seq) + .one(&conn) + .await? + }; + + if let Some(upper_seq) = upper_known_seq { + let signature = Signature::try_from(upper_seq.tx.as_ref())?; + gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); + // Reprocess the entire tree if force is true or if the tree has a seq of 0 to keep the current behavior + } else if force || tree.seq > 0 { + gaps.push(TreeGapFill::new(tree.pubkey, None, None)); + } + + if let Some(lower_seq) = lower_known_seq.filter(|seq| seq.seq > 1) { + let signature = Signature::try_from(lower_seq.tx.as_ref())?; + + gaps.push(TreeGapFill::new(tree.pubkey, Some(signature), None)); + } + + for gap in gaps { + if let Err(e) = tree_gap_sender.send(gap).await { + error!("send gap: {:?}", e); + } + } + } + + drop(tree_gap_sender); + + futures::future::try_join4( + gap_worker, + signature_worker, + program_transformer_worker, + metadata_json_download_worker, + ) + .await?; + + Ok(()) + }) + } +} diff --git a/bubblegum/src/error.rs b/bubblegum/src/error.rs new file mode 100644 index 000000000..420a15a52 --- /dev/null +++ b/bubblegum/src/error.rs @@ -0,0 +1,19 @@ +#[derive(Debug, thiserror::Error)] +pub enum ErrorKind { + #[error("anchor")] + Anchor(#[from] anchor_client::anchor_lang::error::Error), + #[error("solana rpc")] + Rpc(#[from] solana_client::client_error::ClientError), + #[error("parse pubkey")] + ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), + #[error("serialize tree response")] + SerializeTreeResponse, + #[error("sea orm")] + Database(#[from] sea_orm::DbErr), + #[error("try from pubkey")] + TryFromPubkey, + #[error("try from signature")] + TryFromSignature, + #[error("generic error: {0}")] + Generic(String), +} diff --git a/bubblegum/src/lib.rs b/bubblegum/src/lib.rs new file mode 100644 index 000000000..9f800f128 --- /dev/null +++ b/bubblegum/src/lib.rs @@ -0,0 +1,191 @@ +mod backfill; +mod error; +mod tree; + +use das_core::DownloadMetadataJsonRetryConfig; +use das_core::{MetadataJsonDownloadWorkerArgs, Rpc}; +pub use error::ErrorKind; +mod verify; +pub use verify::ProofReport; + +use anyhow::Result; +use backfill::worker::{ProgramTransformerWorkerArgs, SignatureWorkerArgs, TreeWorkerArgs}; +use clap::Parser; +use digital_asset_types::dao::cl_audits_v2; +use futures::{stream::FuturesUnordered, StreamExt}; +use sea_orm::ColumnTrait; +use sea_orm::QueryOrder; +use sea_orm::SqlxPostgresConnector; +use sea_orm::{EntityTrait, QueryFilter}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Signature; +use std::str::FromStr; +use std::sync::Arc; +use tracing::error; +use tree::TreeResponse; + +#[derive(Clone)] +pub struct BubblegumContext { + pub database_pool: sqlx::PgPool, + pub solana_rpc: Rpc, +} + +impl BubblegumContext { + pub const fn new(database_pool: sqlx::PgPool, solana_rpc: Rpc) -> Self { + Self { + database_pool, + solana_rpc, + } + } +} + +#[derive(Debug, Parser, Clone)] +pub struct BackfillArgs { + /// Number of tree crawler workers + #[arg(long, env, default_value = "20")] + pub tree_crawler_count: usize, + + /// The list of trees to crawl. If not specified, all trees will be crawled. + #[arg(long, env, use_value_delimiter = true)] + pub only_trees: Option>, + + #[clap(flatten)] + pub tree_worker: TreeWorkerArgs, +} + +pub async fn start_backfill(context: BubblegumContext, args: BackfillArgs) -> Result<()> { + let trees = if let Some(ref only_trees) = args.only_trees { + TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? + } else { + TreeResponse::all(&context.solana_rpc).await? + }; + + let mut crawl_handles = FuturesUnordered::new(); + + for tree in trees { + if crawl_handles.len() >= args.tree_crawler_count { + crawl_handles.next().await; + } + let context = context.clone(); + let handle = args.tree_worker.start(context, tree); + + crawl_handles.push(handle); + } + + futures::future::try_join_all(crawl_handles).await?; + + Ok(()) +} + +#[derive(Debug, Parser, Clone)] +pub struct BubblegumReplayArgs { + /// The tree to replay. + #[arg(long, env)] + pub tree: String, + + /// The list of sequences to replay. If not specified, all sequences will be replayed. + #[arg(long, env, use_value_delimiter = true)] + pub only_sequences: Option>, + + #[clap(flatten)] + pub signature_worker: SignatureWorkerArgs, + + #[clap(flatten)] + pub program_transformer_worker: ProgramTransformerWorkerArgs, + + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, +} + +pub async fn start_bubblegum_replay( + context: BubblegumContext, + args: BubblegumReplayArgs, +) -> Result<()> { + let pubkey = Pubkey::from_str(&args.tree) + .map(|pubkey| pubkey.to_bytes().to_vec()) + .map_err(|e| anyhow::anyhow!("Invalid tree pubkey: {:?}", e))?; + + let database_pool = context.database_pool.clone(); + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(database_pool); + + let mut query = cl_audits_v2::Entity::find() + .filter(cl_audits_v2::Column::Tree.eq(pubkey)) + .order_by_asc(cl_audits_v2::Column::Seq); + + if let Some(sequences) = args.only_sequences { + query = query.filter(cl_audits_v2::Column::Seq.is_in(sequences)); + } + + let cl_audits = query.all(&conn).await?; + + let metadata_json_download_worker_args = args.metadata_json_download_worker.clone(); + let program_transformer_worker_args = args.program_transformer_worker.clone(); + let signature_worker_args = args.signature_worker.clone(); + + let metadata_json_download_db_pool = context.database_pool.clone(); + let program_transformer_context = context.clone(); + let signature_context = context.clone(); + let download_config = Arc::new(DownloadMetadataJsonRetryConfig::default()); + let (metadata_json_download_worker, metadata_json_download_sender) = + metadata_json_download_worker_args + .start(metadata_json_download_db_pool, download_config)?; + + let (program_transformer_worker, transaction_info_sender) = program_transformer_worker_args + .start(program_transformer_context, metadata_json_download_sender)?; + + let (signature_worker, signature_sender) = + signature_worker_args.start(signature_context, transaction_info_sender)?; + + for audit in cl_audits { + let signature = Signature::try_from(audit.tx.as_ref())?; + if let Err(e) = signature_sender.send(signature).await { + error!("send signature: {:?}", e); + } + } + + drop(signature_sender); + + futures::future::try_join3( + signature_worker, + program_transformer_worker, + metadata_json_download_worker, + ) + .await?; + + Ok(()) +} + +#[derive(Debug, Parser, Clone)] +pub struct VerifyArgs { + /// The list of trees to verify. If not specified, all trees will be crawled. + #[arg(long, env, use_value_delimiter = true)] + pub only_trees: Option>, + + #[arg(long, env, default_value = "20")] + pub max_concurrency: usize, +} + +pub async fn verify_bubblegum( + context: BubblegumContext, + args: VerifyArgs, +) -> Result> { + let trees = if let Some(ref only_trees) = args.only_trees { + TreeResponse::find(&context.solana_rpc, only_trees.clone()).await? + } else { + TreeResponse::all(&context.solana_rpc).await? + }; + + let (sender, receiver) = tokio::sync::mpsc::channel(trees.len()); + + tokio::spawn(async move { + for tree in trees { + if let Ok(report) = verify::check(context.clone(), tree, args.max_concurrency).await { + if sender.send(report).await.is_err() { + error!("Failed to send report"); + } + } + } + }); + + Ok(receiver) +} diff --git a/bubblegum/src/tree.rs b/bubblegum/src/tree.rs new file mode 100644 index 000000000..405c90037 --- /dev/null +++ b/bubblegum/src/tree.rs @@ -0,0 +1,117 @@ +use super::error::ErrorKind; +use anyhow::Result; +use borsh::BorshDeserialize; +use das_core::Rpc; +use solana_client::rpc_filter::{Memcmp, RpcFilterType}; +use solana_sdk::{account::Account, pubkey::Pubkey}; +use spl_account_compression::id; +use spl_account_compression::state::{ + merkle_tree_get_size, ConcurrentMerkleTreeHeader, CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1, +}; +use std::str::FromStr; + +#[derive(Clone)] +pub struct TreeHeaderResponse { + pub max_depth: u32, + pub max_buffer_size: u32, + pub creation_slot: u64, + pub size: usize, +} + +impl TryFrom for TreeHeaderResponse { + type Error = ErrorKind; + + fn try_from(payload: ConcurrentMerkleTreeHeader) -> Result { + let size = merkle_tree_get_size(&payload)?; + + Ok(Self { + max_depth: payload.get_max_depth(), + max_buffer_size: payload.get_max_buffer_size(), + creation_slot: payload.get_creation_slot(), + size, + }) + } +} + +pub struct TreeResponse { + pub pubkey: Pubkey, + pub tree_header: TreeHeaderResponse, + pub seq: u64, +} + +impl TreeResponse { + pub fn try_from_rpc(pubkey: Pubkey, account: Account) -> Result { + let bytes = account.data.as_slice(); + + let (header_bytes, rest) = bytes.split_at(CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1); + let header: ConcurrentMerkleTreeHeader = + ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?; + + let merkle_tree_size = merkle_tree_get_size(&header)?; + let (tree_bytes, _canopy_bytes) = rest.split_at(merkle_tree_size); + + let seq_bytes = tree_bytes[0..8].try_into()?; + let seq = u64::from_le_bytes(seq_bytes); + + let (auth, _) = Pubkey::find_program_address(&[pubkey.as_ref()], &mpl_bubblegum::ID); + + header.assert_valid_authority(&auth)?; + + let tree_header = header.try_into()?; + + Ok(Self { + pubkey, + tree_header, + seq, + }) + } + + pub async fn all(client: &Rpc) -> Result, ErrorKind> { + Ok(client + .get_program_accounts( + &id(), + Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes( + 0, + vec![1u8], + ))]), + ) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Self::try_from_rpc(pubkey, account).ok()) + .collect()) + } + + pub async fn find(client: &Rpc, pubkeys: Vec) -> Result, ErrorKind> { + let pubkeys: Vec = pubkeys + .into_iter() + .map(|p| Pubkey::from_str(&p)) + .collect::, _>>()?; + let pubkey_batches = pubkeys.chunks(100); + let pubkey_batches_count = pubkey_batches.len(); + + let mut gma_handles = Vec::with_capacity(pubkey_batches_count); + + for batch in pubkey_batches { + gma_handles.push(async move { + let accounts = client.get_multiple_accounts(batch).await?; + + let results: Vec<(&Pubkey, Option)> = batch.iter().zip(accounts).collect(); + + Ok::<_, ErrorKind>(results) + }) + } + + let result = futures::future::try_join_all(gma_handles).await?; + + let trees = result + .into_iter() + .flatten() + .filter_map(|(pubkey, account)| { + account.map(|account| Self::try_from_rpc(*pubkey, account)) + }) + .collect::, _>>() + .map_err(|_| ErrorKind::SerializeTreeResponse)?; + + Ok(trees) + } +} diff --git a/bubblegum/src/verify.rs b/bubblegum/src/verify.rs new file mode 100644 index 000000000..e99972ce0 --- /dev/null +++ b/bubblegum/src/verify.rs @@ -0,0 +1,182 @@ +use super::BubblegumContext; +use crate::error::ErrorKind; +use crate::tree::TreeResponse; +use anyhow::{anyhow, Result}; +use digital_asset_types::dapi::get_proof_for_asset; +use digital_asset_types::rpc::AssetProof; +use futures::stream::{FuturesUnordered, StreamExt}; +use mpl_bubblegum::accounts::TreeConfig; +use sea_orm::SqlxPostgresConnector; +use sha3::{Digest, Keccak256}; +use solana_sdk::pubkey::Pubkey; +use spl_account_compression::concurrent_tree_wrapper::ProveLeafArgs; +use std::fmt; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing::debug; + +trait TryFromAssetProof { + fn try_from_asset_proof(proof: AssetProof) -> Result + where + Self: Sized; +} + +impl TryFromAssetProof for ProveLeafArgs { + fn try_from_asset_proof(proof: AssetProof) -> Result { + Ok(ProveLeafArgs { + current_root: bs58::decode(&proof.root) + .into_vec() + .map_err(|e| anyhow!(e))? + .try_into() + .map_err(|_| anyhow!("Invalid root length"))?, + leaf: bs58::decode(&proof.leaf) + .into_vec() + .map_err(|e| anyhow!(e))? + .try_into() + .map_err(|_| anyhow!("Invalid leaf length"))?, + proof_vec: proof + .proof + .iter() + .map(|p| { + bs58::decode(p) + .into_vec() + .map_err(|e| anyhow!(e)) + .and_then(|v| v.try_into().map_err(|_| anyhow!("Invalid proof length"))) + }) + .collect::>>()?, + index: proof.node_index as u32, + }) + } +} + +fn hash(left: &[u8], right: &[u8]) -> [u8; 32] { + let mut hasher = Keccak256::new(); + hasher.update(left); + hasher.update(right); + let result = hasher.finalize(); + let mut hash = [0u8; 32]; + hash.copy_from_slice(&result); + hash +} + +fn verify_merkle_proof(proof: &ProveLeafArgs) -> bool { + let mut node = proof.leaf; + for (i, sibling) in proof.proof_vec.iter().enumerate() { + if (proof.index >> i) & 1 == 0 { + node = hash(&node, sibling); + } else { + node = hash(sibling, &node); + } + } + node == proof.current_root +} + +fn leaf_proof_result(proof: AssetProof) -> Result { + match ProveLeafArgs::try_from_asset_proof(proof) { + Ok(proof) if verify_merkle_proof(&proof) => Ok(ProofResult::Correct), + Ok(_) => Ok(ProofResult::Incorrect), + Err(_) => Ok(ProofResult::Corrupt), + } +} + +#[derive(Debug, Default)] +pub struct ProofReport { + pub tree_pubkey: Pubkey, + pub total_leaves: usize, + pub incorrect_proofs: usize, + pub not_found_proofs: usize, + pub correct_proofs: usize, + pub corrupt_proofs: usize, +} + +#[derive(Debug)] +enum ProofResult { + Correct, + Incorrect, + NotFound, + Corrupt, +} + +impl fmt::Display for ProofResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProofResult::Correct => write!(f, "Correct proof found"), + ProofResult::Incorrect => write!(f, "Incorrect proof found"), + ProofResult::NotFound => write!(f, "Proof not found"), + ProofResult::Corrupt => write!(f, "Corrupt proof found"), + } + } +} + +pub async fn check( + context: BubblegumContext, + tree: TreeResponse, + max_concurrency: usize, +) -> Result { + let (tree_config_pubkey, _) = TreeConfig::find_pda(&tree.pubkey); + + let pool = context.database_pool.clone(); + + let account = context.solana_rpc.get_account(&tree_config_pubkey).await?; + let account = account + .value + .ok_or_else(|| ErrorKind::Generic("Account not found".to_string()))?; + + let tree_config = TreeConfig::from_bytes(account.data.as_slice())?; + + let report = Arc::new(Mutex::new(ProofReport { + tree_pubkey: tree.pubkey, + total_leaves: tree_config.num_minted as usize, + ..ProofReport::default() + })); + + let mut tasks = FuturesUnordered::new(); + + for i in 0..tree_config.num_minted { + if tasks.len() >= max_concurrency { + tasks.next().await; + } + + let db = SqlxPostgresConnector::from_sqlx_postgres_pool(pool.clone()); + let tree_pubkey = tree.pubkey; + let report = Arc::clone(&report); + + tasks.push(tokio::spawn(async move { + let (asset, _) = Pubkey::find_program_address( + &[b"asset", &tree_pubkey.to_bytes(), &i.to_le_bytes()], + &mpl_bubblegum::ID, + ); + let proof_lookup: Result = + get_proof_for_asset(&db, asset.to_bytes().to_vec()) + .await + .map_or_else(|_| Ok(ProofResult::NotFound), leaf_proof_result); + + if let Ok(proof_result) = proof_lookup { + let mut report = report.lock().await; + + match proof_result { + ProofResult::Correct => report.correct_proofs += 1, + ProofResult::Incorrect => report.incorrect_proofs += 1, + ProofResult::NotFound => report.not_found_proofs += 1, + ProofResult::Corrupt => report.corrupt_proofs += 1, + } + + debug!( + tree = %tree_pubkey, + leaf_index = i, + asset = %asset, + result = ?proof_result, + "Proof result for asset" + ); + } + })); + } + + while tasks.next().await.is_some() {} + + let final_report = Arc::try_unwrap(report) + .expect("Failed to unwrap Arc") + .into_inner(); + + Ok(final_report) +} diff --git a/core/Cargo.toml b/core/Cargo.toml index d0532684f..b97e38235 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -8,18 +8,35 @@ publish.workspace = true [dependencies] anyhow = { workspace = true } backon = { workspace = true } +borsh = { workspace = true } +bs58 = { workspace = true } +cadence = { workspace = true } +cadence-macros = { workspace = true } +clap = { workspace = true, features = ["derive", "cargo", "env"] } +derive_more = { workspace = true } +digital_asset_types = { workspace = true } +figment = { workspace = true } +futures = { workspace = true } +indicatif = { workspace = true } +log = { workspace = true } +plerkle_messenger = { workspace = true } +reqwest = { workspace = true } +sea-orm = { workspace = true, features = [ + "sqlx-postgres", + "with-chrono", + "runtime-tokio-rustls", +] } +serde = { workspace = true } +serde_json = { workspace = true } +spl-account-compression = { workspace = true } solana-account-decoder = { workspace = true } solana-client = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } -cadence = { workspace = true } -cadence-macros = { workspace = true } +sqlx = { workspace = true, fatures = ["runtime-tokio-rustls", "postgres"] } thiserror = { workspace = true } -figment = { workspace = true } -plerkle_messenger = { workspace = true } tokio = { workspace = true } -clap = { workspace = true, features = ["derive", "cargo", "env"] } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } +url = { workspace = true } [lints] workspace = true diff --git a/core/src/db.rs b/core/src/db.rs index 3037f670d..5c211a877 100644 --- a/core/src/db.rs +++ b/core/src/db.rs @@ -27,7 +27,7 @@ pub struct PoolArgs { ///// # Returns ///// ///// * `Result` - On success, returns a `DatabaseConnection`. On failure, returns a `DbErr`. -pub async fn connect_db(config: PoolArgs) -> Result { +pub async fn connect_db(config: &PoolArgs) -> Result { let options: PgConnectOptions = config.database_url.parse()?; PgPoolOptions::new() diff --git a/core/src/lib.rs b/core/src/lib.rs index da6bb050e..341c54817 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,9 +1,11 @@ mod db; +mod metadata_json; mod metrics; mod plerkle_messenger_queue; mod solana_rpc; pub use db::*; +pub use metadata_json::*; pub use metrics::*; pub use plerkle_messenger_queue::*; pub use solana_rpc::*; diff --git a/core/src/metadata_json.rs b/core/src/metadata_json.rs new file mode 100644 index 000000000..731d356b5 --- /dev/null +++ b/core/src/metadata_json.rs @@ -0,0 +1,295 @@ +use { + backon::{ExponentialBuilder, Retryable}, + clap::Parser, + digital_asset_types::dao::asset_data, + futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}, + indicatif::HumanDuration, + log::{debug, error}, + reqwest::{Client, Url as ReqwestUrl}, + sea_orm::{entity::*, SqlxPostgresConnector}, + serde::{Deserialize, Serialize}, + std::{sync::Arc, time::Duration}, + tokio::{ + sync::mpsc::{unbounded_channel, UnboundedSender}, + task::JoinHandle, + time::Instant, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadMetadataInfo { + pub asset_data_id: Vec, + pub uri: String, + pub slot: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadMetadataJsonRetryConfig { + pub retries: u8, + pub retry_max_delay_ms: u64, + pub retry_min_delay_ms: u64, +} + +impl Default for DownloadMetadataJsonRetryConfig { + fn default() -> Self { + Self { + retries: 1, + retry_max_delay_ms: 10, + retry_min_delay_ms: 1, + } + } +} + +impl DownloadMetadataJsonRetryConfig { + pub const fn new(retries: usize, retry_max_delay_ms: usize, retry_min_delay_ms: usize) -> Self { + Self { + retries: retries as u8, + retry_max_delay_ms: retry_max_delay_ms as u64, + retry_min_delay_ms: retry_min_delay_ms as u64, + } + } +} + +impl DownloadMetadataInfo { + pub fn new(asset_data_id: Vec, uri: String, slot: i64) -> Self { + Self { + asset_data_id, + uri: uri.trim().replace('\0', ""), + slot, + } + } + + pub fn into_inner(self) -> (Vec, String, i64) { + (self.asset_data_id, self.uri, self.slot) + } +} + +pub type DownloadMetadataNotifier = Box< + dyn Fn( + DownloadMetadataInfo, + ) -> BoxFuture<'static, Result<(), Box>> + + Sync + + Send, +>; + +pub async fn create_download_metadata_notifier( + download_metadata_json_sender: UnboundedSender, +) -> DownloadMetadataNotifier { + Box::new(move |info: DownloadMetadataInfo| -> BoxFuture<'static, Result<(), Box>> + { + let task = download_metadata_json_sender.send(info).map_err(Into::into); + + Box::pin(async move { task }) + }) +} + +#[derive(Parser, Clone, Debug)] +pub struct MetadataJsonDownloadWorkerArgs { + /// The number of worker threads + #[arg(long, env, default_value = "25")] + pub metadata_json_download_worker_count: usize, + /// The request timeout in milliseconds + #[arg(long, env, default_value = "1000")] + pub metadata_json_download_worker_request_timeout: u64, +} + +impl MetadataJsonDownloadWorkerArgs { + pub fn start( + &self, + pool: sqlx::PgPool, + config: Arc, + ) -> Result< + (JoinHandle<()>, UnboundedSender), + MetadataJsonDownloadWorkerError, + > { + let (sender, mut rx) = unbounded_channel::(); + let worker_count = self.metadata_json_download_worker_count; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_millis( + self.metadata_json_download_worker_request_timeout, + )) + .build()?; + let handle = tokio::spawn(async move { + let mut handlers = FuturesUnordered::new(); + let download_config = Arc::clone(&config); + + while let Some(download_metadata_info) = rx.recv().await { + if handlers.len() >= worker_count { + handlers.next().await; + } + + let pool = pool.clone(); + let client = client.clone(); + + handlers.push(spawn_task( + client, + pool, + download_metadata_info, + Arc::clone(&download_config), + )); + } + + while handlers.next().await.is_some() {} + }); + + Ok((handle, sender)) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum MetadataJsonDownloadWorkerError { + #[error("send error")] + Send, + #[error("join error: {0}")] + Join(#[from] tokio::task::JoinError), + #[error("reqwest: {0}")] + Reqwest(#[from] reqwest::Error), +} + +fn spawn_task( + client: Client, + pool: sqlx::PgPool, + download_metadata_info: DownloadMetadataInfo, + config: Arc, +) -> JoinHandle<()> { + tokio::spawn(async move { + let timing = Instant::now(); + let asset_data_id = + bs58::encode(download_metadata_info.asset_data_id.clone()).into_string(); + + if let Err(e) = + perform_metadata_json_task(client, pool, &download_metadata_info, config).await + { + error!("Asset {} failed: {}", asset_data_id, e); + } + + debug!( + "Asset {} finished in {}", + asset_data_id, + HumanDuration(timing.elapsed()) + ); + }) +} + +#[derive(thiserror::Error, Debug)] +pub enum FetchMetadataJsonError { + #[error("reqwest: {0}")] + GenericReqwest(#[from] reqwest::Error), + #[error("json parse for url({url}) with {source}")] + Parse { + source: reqwest::Error, + url: ReqwestUrl, + }, + #[error("response {status} for url ({url}) with {source}")] + Response { + source: reqwest::Error, + url: ReqwestUrl, + status: StatusCode, + }, + #[error("url parse: {0}")] + Url(#[from] url::ParseError), +} + +#[derive(Debug, derive_more::Display)] +pub enum StatusCode { + Unknown, + Code(reqwest::StatusCode), +} + +async fn fetch_metadata_json( + client: Client, + metadata_json_url: &str, + config: Arc, +) -> Result { + (|| async { + let url = ReqwestUrl::parse(metadata_json_url)?; + + let response = client.get(url.clone()).send().await?; + + match response.error_for_status() { + Ok(res) => res + .json::() + .await + .map_err(|source| FetchMetadataJsonError::Parse { source, url }), + Err(source) => { + let status = source + .status() + .map(StatusCode::Code) + .unwrap_or(StatusCode::Unknown); + + Err(FetchMetadataJsonError::Response { + source, + url, + status, + }) + } + } + }) + .retry( + &ExponentialBuilder::default() + .with_max_times(config.retries.into()) + .with_min_delay(Duration::from_millis(config.retry_min_delay_ms)) + .with_max_delay(Duration::from_millis(config.retry_max_delay_ms)), + ) + .await +} + +#[derive(thiserror::Error, Debug)] +pub enum MetadataJsonTaskError { + #[error("sea orm: {0}")] + SeaOrm(#[from] sea_orm::DbErr), + #[error("metadata json: {0}")] + Fetch(#[from] FetchMetadataJsonError), + #[error("asset not found in the db")] + AssetNotFound, +} + +pub async fn perform_metadata_json_task( + client: Client, + pool: sqlx::PgPool, + download_metadata_info: &DownloadMetadataInfo, + config: Arc, +) -> Result<(), MetadataJsonTaskError> { + match fetch_metadata_json(client, &download_metadata_info.uri, config).await { + Ok(metadata) => { + let active_model = asset_data::ActiveModel { + id: Set(download_metadata_info.asset_data_id.clone()), + metadata: Set(metadata), + reindex: Set(Some(false)), + ..Default::default() + }; + + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + + active_model.update(&conn).await?; + + Ok(()) + } + Err(e) => Err(MetadataJsonTaskError::Fetch(e)), + } +} + +pub struct DownloadMetadata { + client: Client, + pool: sqlx::PgPool, +} + +impl DownloadMetadata { + pub const fn new(client: Client, pool: sqlx::PgPool) -> Self { + Self { client, pool } + } + + pub async fn handle_download( + &self, + download_metadata_info: &DownloadMetadataInfo, + config: Arc, + ) -> Result<(), MetadataJsonTaskError> { + perform_metadata_json_task( + self.client.clone(), + self.pool.clone(), + download_metadata_info, + config, + ) + .await + } +} diff --git a/core/src/metrics.rs b/core/src/metrics.rs index 9c0d3c531..13bf4e261 100644 --- a/core/src/metrics.rs +++ b/core/src/metrics.rs @@ -14,8 +14,8 @@ pub struct MetricsArgs { pub metrics_prefix: String, } -pub fn setup_metrics(config: MetricsArgs) -> Result<()> { - let host = (config.metrics_host, config.metrics_port); +pub fn setup_metrics(config: &MetricsArgs) -> Result<()> { + let host = (config.metrics_host.clone(), config.metrics_port); let socket = UdpSocket::bind("0.0.0.0:0")?; socket.set_nonblocking(true)?; diff --git a/core/src/plerkle_messenger_queue.rs b/core/src/plerkle_messenger_queue.rs index 8c30d01c0..11b1166ed 100644 --- a/core/src/plerkle_messenger_queue.rs +++ b/core/src/plerkle_messenger_queue.rs @@ -65,7 +65,7 @@ pub struct QueuePool { } impl QueuePool { - pub async fn try_from_config(config: QueueArgs) -> anyhow::Result { + pub async fn try_from_config(config: &QueueArgs) -> anyhow::Result { let size = usize::try_from(config.messenger_queue_connections)?; let (tx, rx) = mpsc::channel(size); diff --git a/core/src/solana_rpc.rs b/core/src/solana_rpc.rs index 71f86f8a6..26e9e1f05 100644 --- a/core/src/solana_rpc.rs +++ b/core/src/solana_rpc.rs @@ -3,13 +3,16 @@ use backon::ExponentialBuilder; use backon::Retryable; use clap::Parser; use solana_account_decoder::UiAccountEncoding; -use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; +use solana_client::rpc_response::RpcTokenAccountBalance; use solana_client::{ client_error::ClientError, nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcTransactionConfig}, rpc_filter::RpcFilterType, + rpc_request::RpcRequest, + rpc_response::Response as RpcResponse, + rpc_response::RpcConfirmedTransactionStatusWithSignature, }; use solana_sdk::{ account::Account, @@ -31,8 +34,8 @@ pub struct SolanaRpcArgs { pub struct Rpc(Arc); impl Rpc { - pub fn from_config(config: SolanaRpcArgs) -> Self { - Rpc(Arc::new(RpcClient::new(config.solana_rpc_url))) + pub fn from_config(config: &SolanaRpcArgs) -> Self { + Rpc(Arc::new(RpcClient::new(config.solana_rpc_url.clone()))) } pub async fn get_transaction( @@ -157,4 +160,26 @@ impl Rpc { .await? .value) } + + pub async fn get_token_largest_account(&self, mint: Pubkey) -> anyhow::Result { + Ok((|| async { + self.0 + .send::>>( + RpcRequest::Custom { + method: "getTokenLargestAccounts", + }, + serde_json::json!([mint.to_string(),]), + ) + .await + }) + .retry(&ExponentialBuilder::default()) + .await? + .value + .first() + .ok_or(anyhow::anyhow!(format!( + "no token accounts for mint {mint}: burned nft?" + )))? + .address + .parse::()?) + } } diff --git a/digital_asset_types/src/dapi/change_logs.rs b/digital_asset_types/src/dapi/change_logs.rs index 7023ad12f..4742696f0 100644 --- a/digital_asset_types/src/dapi/change_logs.rs +++ b/digital_asset_types/src/dapi/change_logs.rs @@ -200,18 +200,18 @@ fn build_asset_proof( tree_id: Vec, leaf_node_idx: i64, leaf_hash: Vec, - req_indexes: &Vec, + req_indexes: &[i64], required_nodes: &[SimpleChangeLog], ) -> AssetProof { let mut final_node_list = vec![SimpleChangeLog::default(); req_indexes.len()]; for node in required_nodes.iter() { if node.level < final_node_list.len().try_into().unwrap() { - final_node_list[node.level as usize] = node.to_owned(); + node.clone_into(&mut final_node_list[node.level as usize]) } } for (i, (n, nin)) in final_node_list .iter_mut() - .zip(req_indexes.clone()) + .zip(req_indexes.to_owned()) .enumerate() { if *n == SimpleChangeLog::default() { diff --git a/digital_asset_types/tests/common.rs b/digital_asset_types/tests/common.rs index 20989cf45..9b4ba4b15 100644 --- a/digital_asset_types/tests/common.rs +++ b/digital_asset_types/tests/common.rs @@ -31,8 +31,10 @@ pub struct MockMetadataArgs { /// Since we cannot easily change Metadata, we add the new DataV2 fields here at the end. pub token_standard: Option, /// Collection + #[allow(dead_code)] pub collection: Option, /// Uses + #[allow(dead_code)] pub uses: Option, pub creators: Vec, } diff --git a/docker-compose.yaml b/docker-compose.yaml index 4f5bfd93f..7358cb6f5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -121,6 +121,16 @@ services: - "8001:8001" - "8899:8899" - "9900:9900" + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus-config.yaml:/etc/prometheus/prometheus-config.yaml + command: ["--config.file=/etc/prometheus/prometheus-config.yaml"] + ports: + - "9090:9090" + extra_hosts: + - "host.docker.internal:host-gateway" volumes: grafana_data: {} graphite_data: {} diff --git a/grpc-ingest/.gitignore b/grpc-ingest/.gitignore new file mode 100644 index 000000000..191f3badd --- /dev/null +++ b/grpc-ingest/.gitignore @@ -0,0 +1,3 @@ +config-grpc2redis.yaml +config-ingester.yaml +config-monitor.yaml diff --git a/grpc-ingest/Cargo.toml b/grpc-ingest/Cargo.toml new file mode 100644 index 000000000..9cbe506e2 --- /dev/null +++ b/grpc-ingest/Cargo.toml @@ -0,0 +1,66 @@ +[package] +name = "das-grpc-ingest" +version = { workspace = true } +edition = { workspace = true } +repository = { workspace = true } +publish = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +async-stream = { workspace = true } +atty = { workspace = true } +das-bubblegum = { workspace = true } +sqlx = { workspace = true, features = [ + "macros", + "runtime-tokio-rustls", + "postgres", + "uuid", + "offline", + "json", +] } +chrono = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive"] } +thiserror = { workspace = true } +das-core = { workspace = true } +digital_asset_types = { workspace = true } +futures = { workspace = true } +hex = { workspace = true } +hyper = { workspace = true, features = ["server"] } +json5 = { workspace = true } +lazy_static = { workspace = true } +lru = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-jaeger = { workspace = true, features = ["rt-tokio"] } +opentelemetry_sdk = { workspace = true, features = ["trace"] } +program_transformers = { workspace = true } +prometheus = { workspace = true } +redis = { workspace = true, features = ["tokio-comp", "tokio-native-tls-comp"] } +reqwest = { workspace = true } +rust-crypto = { workspace = true } +sea-orm = { workspace = true, features = ["sqlx-postgres"] } +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +solana-sdk = { workspace = true } # only prom rn +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "time", + "fs", + "tracing", +] } +tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } +yellowstone-grpc-client = { workspace = true } +yellowstone-grpc-proto = { workspace = true } +yellowstone-grpc-tools = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +cargo-lock = { workspace = true } +git-version = { workspace = true } +vergen = { workspace = true, features = ["build", "rustc"] } + +[lints] +workspace = true diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md new file mode 100644 index 000000000..1f3a46393 --- /dev/null +++ b/grpc-ingest/README.md @@ -0,0 +1,54 @@ +## Dev setup + +> **Note:** Run these commands from the root of the project + +### Run redis, postgres and prometheus docker containers + +```bash +docker compose up db redis prometheus +``` + +### Seed the database and run migrations + +```bash +INIT_FILE_PATH=./init.sql sea migrate up --database-url=postgres://solana:solana@localhost:5432/solana +``` + +### Configs + +Example config files are available at +- [./config-grpc2redis.example.yml](./config-grpc2redis.example.yml) +- [./config-ingester.example.yml](./config-ingester.example.yml) +- [./config-monitor.example.yml](./config-monitor.example.yml) + +Copy these files and modify them as needed to setup the project. + + +### Run grpc2redis service + +This service will listen to geyser gRPC account and transaction updates. It makes multiple subscriptions to the gRPC stream and filter the data based on the config. The data (vec of bytes) is pushed to a pipeline and then flushed to redis at regular intervals. + +> **Note:** Log level can be set to `info`, `debug`, `warn`, `error` + +```bash +RUST_LOG=info cargo run --bin das-grpc-ingest -- --config grpc-ingest/config-grpc2redis.yml grpc2redis +``` + +### Config for Ingester [./config-ingester.yml](./config-ingester.yml) + +### Run the Ingester service + +This service performs many concurrent tasks + +- Fetch account updates from redis and process them using using program_transformer +- Fetch transaction updates from redis and processe them +- Fetch snapshots from redis and process them +- download token metedata json and store them in postgres db + +```bash + RUST_LOG=debug,sqlx=warn cargo run --bin das-grpc-ingest -- --config grpc-ingest/config-ingester.yml ingester +``` + +### Metrics + +Both grpc2redis and ingester services expose prometheus metrics and can be accessed at `http://localhost:9090/metrics` diff --git a/grpc-ingest/build.rs b/grpc-ingest/build.rs new file mode 100644 index 000000000..92e1f4c7c --- /dev/null +++ b/grpc-ingest/build.rs @@ -0,0 +1,38 @@ +use {cargo_lock::Lockfile, std::collections::HashSet}; + +fn main() -> anyhow::Result<()> { + let mut envs = vergen::EmitBuilder::builder(); + envs.all_build().all_rustc(); + envs.emit()?; + + // vergen git version does not looks cool + println!( + "cargo:rustc-env=GIT_VERSION={}", + git_version::git_version!() + ); + + // Extract packages version + let lockfile = Lockfile::load("../Cargo.lock")?; + println!( + "cargo:rustc-env=SOLANA_SDK_VERSION={}", + get_pkg_version(&lockfile, "solana-sdk") + ); + println!( + "cargo:rustc-env=YELLOWSTONE_GRPC_PROTO_VERSION={}", + get_pkg_version(&lockfile, "yellowstone-grpc-proto") + ); + + Ok(()) +} + +fn get_pkg_version(lockfile: &Lockfile, pkg_name: &str) -> String { + lockfile + .packages + .iter() + .filter(|pkg| pkg.name.as_str() == pkg_name) + .map(|pkg| pkg.version.to_string()) + .collect::>() + .into_iter() + .collect::>() + .join(",") +} diff --git a/grpc-ingest/config-grpc2redis.example.yml b/grpc-ingest/config-grpc2redis.example.yml new file mode 100644 index 000000000..faafa003f --- /dev/null +++ b/grpc-ingest/config-grpc2redis.example.yml @@ -0,0 +1,71 @@ +# This file is used to configure the grpc2redis service. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8873 + +# gRPC server configuration (change x_token and endpoint to the correct ones) +geyser: + # endpoint of the dragonmouth stream + endpoint: http://127.0.0.1:10000 + # x-token of the dragonmouth stream + x_token: null + # transaction commitment level: finalized, confirmed, processed + commitment: finalized + # connection timeout in seconds + connection_timeout: 10 + # request timeout in seconds + timeout: 10 + +# gRPC subscription configuration (each representing a separate stream processed concurrently) +# check account and transaction filters here (https://github.com/rpcpool/yellowstone-grpc/blob/master/README.md) +subscriptions: + metadata: + stream: + # stream name + name: ACCOUNTS + # maximum length of the stream. + max_len: 100_000_000 + # maximum concurrency for processing the stream. + max_concurrency: 2 + # maximum idle time (in milliseconds) before the redis pipeline is flushed + pipeline_max_idle_ms: 150 + # number of pipelines that can be processed concurrently + pipeline_count: 1 + # filter accounts by owner + filter: + accounts: + owner: + - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s + token: + stream: + name: ACCOUNTS + max_len: 100_000_000 + max_concurrency: 5 + filter: + accounts: + owner: + - TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA + core: + stream: + name: ACCOUNTS + max_len: 100_000_000 + max_concurrency: 2 + filter: + accounts: + owner: + - CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d + bubblegum: + stream: + name: TRANSACTIONS + max_len: 100_000_000 + max_concurrency: 2 + # filter transactions by accounts_included + filter: + transactions: + account_include: + - BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY + +# Redis configuration +redis: + # redis connection url + url: redis://localhost:6379 diff --git a/grpc-ingest/config-ingester.example.yml b/grpc-ingest/config-ingester.example.yml new file mode 100644 index 000000000..c72bff163 --- /dev/null +++ b/grpc-ingest/config-ingester.example.yml @@ -0,0 +1,61 @@ +# This file is used to configure the ingester service. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8875 + +# redis connection url +redis: redis://localhost:6379 + +# postgres configuration +postgres: + # connection url + url: postgres://solana:solana@localhost/solana + # min connections + min_connections: 10 + # max connections + max_connections: 50 + +# snapshots configuration for the ingester +snapshots: + # stream name + name: SNAPSHOTS + # maximum number of concurrent tasks for processing. + max_concurrency: 10 + # maximum number of elements to return per redis stream + batch_size: 100 + # maximum idle time (in milliseconds) before processing a batch + xack_batch_max_idle_ms: 1_000 + # buffer size for redis acknowledgment messages + xack_buffer_size: 10_000 + # maximum size of a batch for redis acknowledgment processing. + xack_batch_max_size: 500 + # maximum number of redis messages to keep to buffer. + messages_buffer_size: 200 + # redis stream consumer name must be unique per ingester process + consumer: consumer + +# accounts configuration for the ingester +accounts: + name: ACCOUNTS + max_concurrency: 10 + batch_size: 100 + xack_batch_max_idle_ms: 1_000 + xack_buffer_size: 10_000 + xack_batch_max_size: 500 + +# transactions configuration for the ingester +transactions: + name: TRANSACTIONS + +# download metadata configuration for the ingester +download_metadata: + # maximum number of attempts for downloading metadata JSON. + max_attempts: 1 + # reretry max delay in milliseconds + retry_max_delay: 10 + # retry min delay in milliseconds + retry_min_delay: 1 + # request timeout in milliseconds + request_timeout: 5_000 + stream: + name: METADATA_JSON diff --git a/grpc-ingest/config-monitor.example.yml b/grpc-ingest/config-monitor.example.yml new file mode 100644 index 000000000..e2826a9e5 --- /dev/null +++ b/grpc-ingest/config-monitor.example.yml @@ -0,0 +1,20 @@ +# this file is used to configure the monitor service for bubblegum merkle trees. + +# prometheus metrics are pushed to this endpoint +prometheus: 0.0.0.0:8876 + +# rpc endpoint +rpc: http://127.0.0.1:8899 + +# postgres configuration +postgres: + # connection url + url: postgres://solana:solana@localhost/solana + # min connections + min_connections: 10 + # maximum allowed connections + max_connections: 50 + +# bubblegum merkle tree configuration +bubblegum: + only_trees: null diff --git a/grpc-ingest/src/config.rs b/grpc-ingest/src/config.rs new file mode 100644 index 000000000..d778d1c14 --- /dev/null +++ b/grpc-ingest/src/config.rs @@ -0,0 +1,399 @@ +use { + anyhow::Context, + serde::{de, Deserialize}, + std::{collections::HashMap, net::SocketAddr, path::Path, time::Duration}, + tokio::fs, + yellowstone_grpc_tools::config::{ + deserialize_usize_str, ConfigGrpcRequestAccounts, ConfigGrpcRequestCommitment, + ConfigGrpcRequestTransactions, + }, +}; + +pub const REDIS_STREAM_DATA_KEY: &str = "data"; + +pub async fn load(path: impl AsRef + Copy) -> anyhow::Result +where + T: de::DeserializeOwned, +{ + let text = fs::read_to_string(path) + .await + .context("failed to read config from file")?; + + match path.as_ref().extension().and_then(|e| e.to_str()) { + Some("yaml") | Some("yml") => { + serde_yaml::from_str(&text).context("failed to parse config from YAML file") + } + Some("json") => json5::from_str(&text).context("failed to parse config from JSON file"), + value => anyhow::bail!("unknown config extension: {value:?}"), + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigIngestStream { + pub name: String, + #[serde(default = "ConfigIngestStream::default_group")] + pub group: String, + #[serde(default = "ConfigIngestStream::default_consumer")] + pub consumer: String, + #[serde( + default = "ConfigIngestStream::default_xack_batch_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub xack_batch_max_size: usize, + #[serde( + default = "ConfigIngestStream::default_xack_batch_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "xack_batch_max_idle_ms" + )] + pub xack_batch_max_idle: Duration, + #[serde( + default = "ConfigIngestStream::default_batch_size", + deserialize_with = "deserialize_usize_str" + )] + pub batch_size: usize, + #[serde( + default = "ConfigIngestStream::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, + #[serde( + default = "ConfigIngestStream::default_ack_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub ack_concurrency: usize, + #[serde( + default = "ConfigIngestStream::default_xack_buffer_size", + deserialize_with = "deserialize_usize_str" + )] + pub xack_buffer_size: usize, + #[serde( + default = "ConfigIngestStream::default_message_buffer_size", + deserialize_with = "deserialize_usize_str" + )] + pub message_buffer_size: usize, +} + +impl ConfigIngestStream { + pub const fn default_xack_buffer_size() -> usize { + 1_000 + } + + pub const fn default_message_buffer_size() -> usize { + 100 + } + + pub const fn default_max_concurrency() -> usize { + 2 + } + + pub const fn default_ack_concurrency() -> usize { + 5 + } + + pub const fn default_xack_batch_max_idle() -> Duration { + Duration::from_millis(10_000) + } + + pub fn default_group() -> String { + "ingester".to_owned() + } + + pub fn default_consumer() -> String { + "consumer".to_owned() + } + + pub const fn default_xack_batch_max_size() -> usize { + 5 + } + + pub const fn default_batch_size() -> usize { + 5 + } +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default)] +pub struct ConfigPrometheus { + pub prometheus: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGeyser { + pub endpoint: String, + pub x_token: Option, + #[serde(default = "ConfigGeyser::default_commitment")] + pub commitment: ConfigGrpcRequestCommitment, + #[serde(default = "ConfigGeyser::default_connection_timeout")] + pub connect_timeout: u64, + #[serde(default = "ConfigGeyser::default_timeout")] + pub timeout: u64, +} + +impl ConfigGeyser { + pub const fn default_commitment() -> ConfigGrpcRequestCommitment { + ConfigGrpcRequestCommitment::Finalized + } + + pub const fn default_connection_timeout() -> u64 { + 10 + } + + pub const fn default_timeout() -> u64 { + 10 + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigStream { + pub name: String, + #[serde( + default = "ConfigStream::default_stream_maxlen", + deserialize_with = "deserialize_usize_str" + )] + pub max_len: usize, + #[serde( + default = "ConfigStream::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, + #[serde( + default = "ConfigStream::default_pipeline_count", + deserialize_with = "deserialize_usize_str" + )] + pub pipeline_count: usize, + #[serde( + default = "ConfigStream::default_pipeline_max_idle", + deserialize_with = "deserialize_duration_str" + )] + pub pipeline_max_idle: Duration, +} + +impl ConfigStream { + pub const fn default_stream_maxlen() -> usize { + 10_000_000 + } + + pub const fn default_max_concurrency() -> usize { + 10 + } + + pub const fn default_pipeline_count() -> usize { + 1 + } + + pub const fn default_pipeline_max_idle() -> Duration { + Duration::from_millis(150) + } +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGrpcRequestFilter { + pub accounts: Option, + pub transactions: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigSubscription { + pub stream: ConfigStream, + pub filter: ConfigGrpcRequestFilter, +} + +pub type ConfigGrpcSubscriptions = HashMap; + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGrpc { + pub geyser: ConfigGeyser, + + pub subscriptions: ConfigGrpcSubscriptions, + + pub redis: ConfigGrpcRedis, +} + +#[derive(Debug, Clone, Deserialize, Default)] +pub struct ConfigGrpcRedis { + pub url: String, +} + +pub fn deserialize_duration_str<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + let ms = deserialize_usize_str(deserializer)?; + Ok(Duration::from_millis(ms as u64)) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigIngester { + pub redis: String, + pub postgres: ConfigPostgres, + pub download_metadata: ConfigIngesterDownloadMetadata, + pub snapshots: ConfigIngestStream, + pub accounts: ConfigIngestStream, + pub transactions: ConfigIngestStream, +} + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ConfigIngesterRedisStreamType { + Account, + Transaction, + MetadataJson, + Snapshot, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigPostgres { + pub url: String, + #[serde( + default = "ConfigPostgres::default_min_connections", + deserialize_with = "deserialize_usize_str" + )] + pub min_connections: usize, + #[serde( + default = "ConfigPostgres::default_max_connections", + deserialize_with = "deserialize_usize_str" + )] + pub max_connections: usize, + #[serde( + default = "ConfigPostgres::default_idle_timeout", + deserialize_with = "deserialize_duration_str" + )] + pub idle_timeout: Duration, + #[serde( + default = "ConfigPostgres::default_max_lifetime", + deserialize_with = "deserialize_duration_str" + )] + pub max_lifetime: Duration, +} + +impl ConfigPostgres { + pub const fn default_min_connections() -> usize { + 10 + } + + pub const fn default_max_connections() -> usize { + 50 + } + + pub const fn default_idle_timeout() -> Duration { + Duration::from_millis(75) + } + + pub const fn default_max_lifetime() -> Duration { + Duration::from_millis(125) + } +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ConfigIngesterDownloadMetadata { + pub stream: ConfigIngestStream, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_num_threads", + deserialize_with = "deserialize_usize_str" + )] + pub _num_threads: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_max_attempts", + deserialize_with = "deserialize_usize_str" + )] + pub max_attempts: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_request_timeout", + deserialize_with = "deserialize_duration_str", + rename = "request_timeout_ms" + )] + pub request_timeout: Duration, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_stream_maxlen", + deserialize_with = "deserialize_usize_str" + )] + pub stream_maxlen: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_stream_max_size", + deserialize_with = "deserialize_usize_str" + )] + pub _pipeline_max_size: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_pipeline_max_idle", + deserialize_with = "deserialize_duration_str", + rename = "pipeline_max_idle_ms" + )] + pub _pipeline_max_idle: Duration, + + #[serde( + default = "ConfigIngesterDownloadMetadata::default_retry_max_delay_ms", + deserialize_with = "deserialize_usize_str" + )] + pub retry_max_delay_ms: usize, + #[serde( + default = "ConfigIngesterDownloadMetadata::default_retry_min_delay_ms", + deserialize_with = "deserialize_usize_str" + )] + pub retry_min_delay_ms: usize, +} + +impl ConfigIngesterDownloadMetadata { + pub const fn default_num_threads() -> usize { + 2 + } + + pub const fn default_pipeline_max_idle() -> Duration { + Duration::from_millis(10) + } + + pub const fn default_stream_max_size() -> usize { + 10 + } + + pub const fn default_stream_maxlen() -> usize { + 10_000_000 + } + + pub const fn default_max_attempts() -> usize { + 3 + } + + pub const fn default_request_timeout() -> Duration { + Duration::from_millis(3_000) + } + + pub const fn default_retry_max_delay_ms() -> usize { + 10 + } + + pub const fn default_retry_min_delay_ms() -> usize { + 1 + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigMonitor { + pub postgres: ConfigPostgres, + pub rpc: String, + pub bubblegum: ConfigBubblegumVerify, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ConfigBubblegumVerify { + #[serde( + default = "ConfigBubblegumVerify::default_report_interval", + deserialize_with = "deserialize_duration_str" + )] + pub _report_interval: Duration, + #[serde(default)] + pub only_trees: Option>, + #[serde( + default = "ConfigBubblegumVerify::default_max_concurrency", + deserialize_with = "deserialize_usize_str" + )] + pub max_concurrency: usize, +} + +impl ConfigBubblegumVerify { + pub const fn default_report_interval() -> Duration { + Duration::from_millis(5 * 60 * 1000) + } + pub const fn default_max_concurrency() -> usize { + 20 + } +} diff --git a/grpc-ingest/src/grpc.rs b/grpc-ingest/src/grpc.rs new file mode 100644 index 000000000..c0377a8be --- /dev/null +++ b/grpc-ingest/src/grpc.rs @@ -0,0 +1,372 @@ +use { + crate::{ + config::{ConfigGrpc, ConfigGrpcRequestFilter, ConfigSubscription}, + prom::{grpc_tasks_total_dec, grpc_tasks_total_inc, redis_xadd_status_inc}, + redis::TrackedPipeline, + util::create_shutdown, + }, + anyhow::Context, + futures::{ + stream::{FuturesUnordered, StreamExt}, + SinkExt, + }, + redis::streams::StreamMaxlen, + std::{collections::HashMap, sync::Arc, time::Duration}, + tokio::{ + sync::{oneshot, Mutex}, + time::sleep, + }, + tracing::{debug, warn}, + yellowstone_grpc_client::GeyserGrpcClient, + yellowstone_grpc_proto::{ + geyser::{SubscribeRequest, SubscribeRequestPing, SubscribeUpdate}, + prelude::subscribe_update::UpdateOneof, + prost::Message, + }, + yellowstone_grpc_tools::config::GrpcRequestToProto, +}; + +const PING_ID: i32 = 0; + +pub async fn run(config: ConfigGrpc) -> anyhow::Result<()> { + let redis_client = redis::Client::open(config.redis.url.clone())?; + let connection = redis_client.get_multiplexed_tokio_connection().await?; + + let mut shutdown = create_shutdown()?; + + let config = Arc::new(config); + + let subscriptions = config.subscriptions.clone(); + + let (global_shutdown_tx, mut global_shutdown_rx) = oneshot::channel(); + let global_shutdown_tx = Arc::new(Mutex::new(Some(global_shutdown_tx))); + + let mut subscription_tasks = Vec::new(); + for (label, subscription_config) in subscriptions { + let subscription = Subscription { + label, + config: subscription_config, + }; + let task = SubscriptionTask::build() + .config(Arc::clone(&config)) + .connection(connection.clone()) + .subscription(subscription) + .start(Arc::clone(&global_shutdown_tx)) + .await?; + + subscription_tasks.push(task); + } + + tokio::select! { + _ = &mut global_shutdown_rx => { + warn!( + target: "grpc2redis", + action = "global_shutdown_signal_received", + message = "Global shutdown signal received, stopping all tasks" + ); + } + _ = shutdown.next() => { + warn!( + target: "grpc2redis", + action = "shutdown_signal_received", + message = "Shutdown signal received, waiting for spawned tasks to complete" + ); + } + } + + futures::future::join_all( + subscription_tasks + .into_iter() + .map(|task| task.stop()) + .collect::>(), + ) + .await + .into_iter() + .collect::>()?; + + Ok(()) +} + +pub struct Subscription { + pub label: String, + pub config: ConfigSubscription, +} + +#[derive(Default)] +pub struct SubscriptionTask { + pub config: Arc, + pub connection: Option, + pub subscription: Option, +} + +impl SubscriptionTask { + pub fn build() -> Self { + Self::default() + } + + pub fn config(mut self, config: Arc) -> Self { + self.config = config; + self + } + + pub fn subscription(mut self, subscription: Subscription) -> Self { + self.subscription = Some(subscription); + self + } + + pub fn connection(mut self, connection: redis::aio::MultiplexedConnection) -> Self { + self.connection = Some(connection); + self + } + + pub async fn start( + mut self, + global_shutdown_tx: Arc>>>, + ) -> anyhow::Result { + let config = Arc::clone(&self.config); + let connection = self + .connection + .take() + .expect("Redis Connection is required"); + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let subscription = self.subscription.take().expect("Subscription is required"); + let label = subscription.label.clone(); + let subscription_config = Arc::new(subscription.config); + let connection = connection.clone(); + + let ConfigSubscription { stream, filter } = subscription_config.as_ref().clone(); + + let stream_config = Arc::new(stream.clone()); + let mut req_accounts = HashMap::with_capacity(1); + let mut req_transactions = HashMap::with_capacity(1); + + let ConfigGrpcRequestFilter { + accounts, + transactions, + } = filter; + + if let Some(accounts) = accounts { + req_accounts.insert(label.clone(), accounts.to_proto()); + } + + if let Some(transactions) = transactions { + req_transactions.insert(label.clone(), transactions.to_proto()); + } + + let request = SubscribeRequest { + accounts: req_accounts, + transactions: req_transactions, + commitment: Some(config.geyser.commitment.to_proto().into()), + ..Default::default() + }; + + let pipes: Vec<_> = (0..stream_config.pipeline_count) + .map(|_| Arc::new(Mutex::new(TrackedPipeline::default()))) + .collect(); + let mut tasks = FuturesUnordered::new(); + + let mut dragon_mouth_client = + GeyserGrpcClient::build_from_shared(config.geyser.endpoint.clone())? + .x_token(config.geyser.x_token.clone())? + .connect_timeout(Duration::from_secs(config.geyser.connect_timeout)) + .timeout(Duration::from_secs(config.geyser.timeout)) + .connect() + .await + .context("failed to connect to gRPC")?; + + let (mut subscribe_tx, stream) = dragon_mouth_client + .subscribe_with_request(Some(request)) + .await?; + let global_shutdown_tx = Arc::clone(&global_shutdown_tx); + + let control = tokio::spawn({ + async move { + tokio::pin!(stream); + + let mut flush_handles = Vec::new(); + let mut shutdown_senders = Vec::new(); + + for pipe in &pipes { + let pipe = Arc::clone(pipe); + let stream_config = Arc::clone(&stream_config); + let label = label.clone(); + let mut connection = connection.clone(); + let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); + + let flush_handle = tokio::spawn(async move { + loop { + tokio::select! { + _ = sleep(stream_config.pipeline_max_idle) => { + let mut pipe = pipe.lock().await; + let flush = pipe.flush(&mut connection).await; + + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let count = flush.as_ref().unwrap_or_else(|count| count); + + debug!(target: "grpc2redis", action = "flush_redis_pip_deadline", stream = ?stream_config.name, status = ?status, count = ?count); + redis_xadd_status_inc(&stream_config.name, &label, status, *count); + } + _ = &mut shutdown_rx => { + let mut pipe = pipe.lock().await; + let flush = pipe.flush(&mut connection).await; + + let status = flush.as_ref().map(|_| ()).map_err(|_| ()); + let count = flush.as_ref().unwrap_or_else(|count| count); + + debug!(target: "grpc2redis", action = "final_flush_redis_pipe", stream = ?stream_config.name, status = ?status, count = ?count); + redis_xadd_status_inc(&stream_config.name, &label, status, *count); + break; + } + } + } + }); + + flush_handles.push(flush_handle); + shutdown_senders.push(shutdown_tx); + } + + let mut current_pipe_index = 0; + + loop { + tokio::select! { + event = stream.next() => { + match event { + Some(Ok(msg)) => { + match msg.update_oneof { + Some(UpdateOneof::Account(_)) | Some(UpdateOneof::Transaction(_)) => { + if tasks.len() >= stream_config.max_concurrency { + tasks.next().await; + } + grpc_tasks_total_inc(&label, &stream_config.name); + + tasks.push(tokio::spawn({ + let pipe = Arc::clone(&pipes[current_pipe_index]); + let label = label.clone(); + let stream_config = Arc::clone(&stream_config); + + async move { + let stream = stream_config.name.clone(); + let stream_maxlen = stream_config.max_len; + + let SubscribeUpdate { update_oneof, .. } = msg; + + let mut pipe = pipe.lock().await; + + if let Some(update) = update_oneof { + match update { + UpdateOneof::Account(account) => { + pipe.xadd_maxlen( + &stream.to_string(), + StreamMaxlen::Approx(stream_maxlen), + "*", + account.encode_to_vec(), + ); + } + + UpdateOneof::Transaction(transaction) => { + pipe.xadd_maxlen( + &stream.to_string(), + StreamMaxlen::Approx(stream_maxlen), + "*", + transaction.encode_to_vec(), + ); + } + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant", label = ?label, message = "Unknown update variant"); + } + } + } + + grpc_tasks_total_dec(&label, &stream_config.name); + } + })); + + current_pipe_index = (current_pipe_index + 1) % pipes.len(); + } + Some(UpdateOneof::Ping(_)) => { + let ping = subscribe_tx + .send(SubscribeRequest { + ping: Some(SubscribeRequestPing { id: PING_ID }), + ..Default::default() + }) + .await; + + match ping { + Ok(_) => { + debug!(target: "grpc2redis", action = "send_ping", message = "Ping sent successfully", id = PING_ID); + } + Err(err) => { + warn!(target: "grpc2redis", action = "send_ping_failed", message = "Failed to send ping", ?err, id = PING_ID); + } + } + } + Some(UpdateOneof::Pong(pong)) => { + if pong.id == PING_ID { + debug!(target: "grpc2redis", action = "receive_pong", message = "Pong received", id = PING_ID); + } else { + warn!(target: "grpc2redis", action = "receive_unknown_pong", message = "Unknown pong id received", id = pong.id); + } + } + _ => { + warn!(target: "grpc2redis", action = "unknown_update_variant", message = "Unknown update variant"); + } + } + } + None => { + warn!(target: "grpc2redis", action = "stream_closed", message = "Stream closed, stopping subscription task", ?label); + + let mut global_shutdown_tx = global_shutdown_tx.lock().await; + if let Some(global_shutdown_tx) = global_shutdown_tx.take() { + let _ = global_shutdown_tx.send(()); + } + } + Some(Err(_)) => { + warn!(target: "grpc2redis", action = "stream_error", message = "Stream error, skipping message", ?label); + } + } + } + _ = &mut shutdown_rx => { + debug!(target: "grpc2redis", action = "shutdown_signal_received", message = "Shutdown signal received, stopping subscription task", ?label); + break; + } + } + } + + debug!(target: "grpc2redis", action = "shutdown_subscription_task", message = "Subscription task stopped", ?label); + while (tasks.next().await).is_some() {} + + for shutdown_tx in shutdown_senders { + debug!(target: "grpc2redis", action = "send_shutdown_signal", message = "Sending shutdown signal to flush handles"); + let _ = shutdown_tx.send(()); + } + + debug!(target: "grpc2redis", action = "wait_flush_handles", message = "Waiting for flush handles to complete"); + futures::future::join_all(flush_handles).await; + } + }); + + Ok(SubscriptionTaskStop { + shutdown_tx, + control, + }) + } +} + +#[derive(Debug)] +pub struct SubscriptionTaskStop { + pub shutdown_tx: tokio::sync::oneshot::Sender<()>, + pub control: tokio::task::JoinHandle<()>, +} + +impl SubscriptionTaskStop { + pub async fn stop(self) -> anyhow::Result<()> { + self.shutdown_tx + .send(()) + .map_err(|_| anyhow::anyhow!("Failed to send shutdown signal"))?; + + self.control.await?; + + Ok(()) + } +} diff --git a/grpc-ingest/src/ingester.rs b/grpc-ingest/src/ingester.rs new file mode 100644 index 000000000..c72c456d0 --- /dev/null +++ b/grpc-ingest/src/ingester.rs @@ -0,0 +1,153 @@ +use { + crate::{ + config::{ConfigIngester, REDIS_STREAM_DATA_KEY}, + postgres::{create_pool as pg_create_pool, report_pgpool}, + prom::redis_xadd_status_inc, + redis::{AccountHandle, DownloadMetadataJsonHandle, IngestStream, TransactionHandle}, + util::create_shutdown, + }, + das_core::{ + DownloadMetadata, DownloadMetadataInfo, DownloadMetadataJsonRetryConfig, + DownloadMetadataNotifier, + }, + futures::{future::BoxFuture, stream::StreamExt}, + program_transformers::ProgramTransformer, + redis::aio::MultiplexedConnection, + std::sync::Arc, + tokio::time::{sleep, Duration}, + tracing::warn, +}; + +fn download_metadata_notifier_v2( + connection: MultiplexedConnection, + stream: String, + stream_maxlen: usize, +) -> anyhow::Result { + Ok( + Box::new( + move |info: DownloadMetadataInfo| -> BoxFuture< + 'static, + Result<(), Box>, + > { + let mut connection = connection.clone(); + let stream = stream.clone(); + Box::pin(async move { + + let info_bytes = serde_json::to_vec(&info)?; + + let xadd = redis::cmd("XADD") + .arg(&stream) + .arg("MAXLEN") + .arg("~") + .arg(stream_maxlen) + .arg("*") + .arg(REDIS_STREAM_DATA_KEY) + .arg(info_bytes) + .query_async::<_, redis::Value>(&mut connection) + .await; + + let status = xadd.map(|_| ()).map_err(|_| ()); + + redis_xadd_status_inc(&stream, "metadata_notifier",status, 1); + + Ok(()) + }) + }, + ), + ) +} + +pub async fn run(config: ConfigIngester) -> anyhow::Result<()> { + let redis_client = redis::Client::open(config.redis)?; + let connection = redis_client.get_multiplexed_tokio_connection().await?; + let pool = pg_create_pool(config.postgres).await?; + + let download_metadata_stream = config.download_metadata.stream.clone(); + let download_metadata_stream_maxlen = config.download_metadata.stream_maxlen; + + let download_metadata_notifier = download_metadata_notifier_v2( + connection.clone(), + download_metadata_stream.name.clone(), + download_metadata_stream_maxlen, + )?; + + let program_transformer = Arc::new(ProgramTransformer::new( + pool.clone(), + download_metadata_notifier, + )); + let http_client = reqwest::Client::builder() + .timeout(config.download_metadata.request_timeout) + .build()?; + + let download_metadata = Arc::new(DownloadMetadata::new(http_client, pool.clone())); + let download_metadatas = IngestStream::build() + .config(config.download_metadata.stream.clone()) + .connection(connection.clone()) + .handler(DownloadMetadataJsonHandle::new( + Arc::clone(&download_metadata), + Arc::new(DownloadMetadataJsonRetryConfig::new( + config.download_metadata.max_attempts, + config.download_metadata.retry_max_delay_ms, + config.download_metadata.retry_min_delay_ms, + )), + )) + .start() + .await?; + + let accounts = IngestStream::build() + .config(config.accounts) + .connection(connection.clone()) + .handler(AccountHandle::new(Arc::clone(&program_transformer))) + .start() + .await?; + + let transactions = IngestStream::build() + .config(config.transactions) + .connection(connection.clone()) + .handler(TransactionHandle::new(Arc::clone(&program_transformer))) + .start() + .await?; + + let snapshots = IngestStream::build() + .config(config.snapshots) + .connection(connection.clone()) + .handler(AccountHandle::new(Arc::clone(&program_transformer))) + .start() + .await?; + + let mut shutdown = create_shutdown()?; + + let report_pool = pool.clone(); + let report = tokio::spawn(async move { + let pool = report_pool.clone(); + loop { + sleep(Duration::from_millis(100)).await; + report_pgpool(pool.clone()); + } + }); + + if let Some(signal) = shutdown.next().await { + warn!( + target: "ingester", + action = "shutdown_signal_received", + message = "Shutdown signal received, waiting for spawned tasks to complete", + signal = ?signal + ); + } + + futures::future::join_all(vec![ + accounts.stop(), + transactions.stop(), + snapshots.stop(), + download_metadatas.stop(), + ]) + .await + .into_iter() + .collect::>()?; + + report.abort(); + + pool.close().await; + + Ok::<(), anyhow::Error>(()) +} diff --git a/grpc-ingest/src/main.rs b/grpc-ingest/src/main.rs new file mode 100644 index 000000000..99ad667be --- /dev/null +++ b/grpc-ingest/src/main.rs @@ -0,0 +1,87 @@ +use { + crate::{ + config::{load as config_load, ConfigGrpc, ConfigIngester, ConfigPrometheus}, + prom::run_server as prometheus_run_server, + tracing::init as tracing_init, + }, + anyhow::Context, + clap::{Parser, Subcommand}, + config::ConfigMonitor, + std::net::SocketAddr, +}; + +mod config; +mod grpc; +mod ingester; +mod monitor; +mod postgres; +mod prom; +mod redis; +mod tracing; +mod util; +mod version; + +#[derive(Debug, Parser)] +#[clap(author, version)] +struct Args { + /// Path to config file + #[clap(short, long)] + config: String, + + /// Prometheus listen address + #[clap(long)] + prometheus: Option, + + #[command(subcommand)] + action: ArgsAction, +} + +#[derive(Debug, Clone, Subcommand)] +enum ArgsAction { + /// Subscribe on Geyser events using gRPC and send them to Redis + #[command(name = "grpc2redis")] + Grpc, + /// Run ingester process (process events from Redis) + #[command(name = "ingester")] + Ingester, + #[command(name = "monitor")] + Monitor, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_init()?; + + let args = Args::parse(); + + // Run prometheus server + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse prometheus config from: {}", args.config))?; + if let Some(address) = args.prometheus.or(config.prometheus) { + prometheus_run_server(address)?; + } + + // Run grpc / ingester / download-metadata + match args.action { + ArgsAction::Grpc => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + grpc::run(config).await + } + ArgsAction::Ingester => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + ingester::run(config).await + } + ArgsAction::Monitor => { + let config = config_load::(&args.config) + .await + .with_context(|| format!("failed to parse config from: {}", args.config))?; + + monitor::run(config).await + } + } +} diff --git a/grpc-ingest/src/monitor.rs b/grpc-ingest/src/monitor.rs new file mode 100644 index 000000000..7c7e277f8 --- /dev/null +++ b/grpc-ingest/src/monitor.rs @@ -0,0 +1,50 @@ +use crate::postgres::create_pool; +use crate::util::create_shutdown; +use crate::{config::ConfigMonitor, prom::update_tree_proof_report}; +use das_bubblegum::{verify_bubblegum, BubblegumContext, VerifyArgs}; +use das_core::{Rpc, SolanaRpcArgs}; +use futures::stream::StreamExt; +use tracing::{error, info}; + +pub async fn run(config: ConfigMonitor) -> anyhow::Result<()> { + let mut shutdown = create_shutdown()?; + let database_pool = create_pool(config.postgres).await?; + let rpc = Rpc::from_config(&SolanaRpcArgs { + solana_rpc_url: config.rpc, + }); + + let bubblegum_verify = tokio::spawn(async move { + loop { + let bubblegum_context = BubblegumContext::new(database_pool.clone(), rpc.clone()); + let verify_args = VerifyArgs { + only_trees: config.bubblegum.only_trees.clone(), + max_concurrency: config.bubblegum.max_concurrency, + }; + + match verify_bubblegum(bubblegum_context, verify_args).await { + Ok(mut reports_receiver) => { + while let Some(report) = reports_receiver.recv().await { + info!( + report = ?report, + ); + update_tree_proof_report(&report); + } + + tokio::time::sleep(tokio::time::Duration::from_secs(600)).await; + } + Err(e) => { + error!( + message = "Error proof report recv", + error = ?e + ); + } + } + } + }); + + if let Some(_signal) = shutdown.next().await {} + + bubblegum_verify.abort(); + + Ok(()) +} diff --git a/grpc-ingest/src/postgres.rs b/grpc-ingest/src/postgres.rs new file mode 100644 index 000000000..d38c5250c --- /dev/null +++ b/grpc-ingest/src/postgres.rs @@ -0,0 +1,27 @@ +use { + crate::{ + config::ConfigPostgres, + prom::{pgpool_connections_set, PgpoolConnectionsKind}, + }, + sqlx::{ + postgres::{PgConnectOptions, PgPoolOptions}, + PgPool, + }, +}; + +pub async fn create_pool(config: ConfigPostgres) -> anyhow::Result { + let options: PgConnectOptions = config.url.parse()?; + PgPoolOptions::new() + .min_connections(config.min_connections.try_into()?) + .max_connections(config.max_connections.try_into()?) + .idle_timeout(config.idle_timeout) + .max_lifetime(config.max_lifetime) + .connect_with(options) + .await + .map_err(Into::into) +} + +pub fn report_pgpool(pgpool: PgPool) { + pgpool_connections_set(PgpoolConnectionsKind::Total, pgpool.size() as usize); + pgpool_connections_set(PgpoolConnectionsKind::Idle, pgpool.num_idle()); +} diff --git a/grpc-ingest/src/prom.rs b/grpc-ingest/src/prom.rs new file mode 100644 index 000000000..b4ff567fe --- /dev/null +++ b/grpc-ingest/src/prom.rs @@ -0,0 +1,404 @@ +use { + crate::{redis::RedisStreamMessageError, version::VERSION as VERSION_INFO}, + das_bubblegum::ProofReport, + das_core::MetadataJsonTaskError, + hyper::{ + server::conn::AddrStream, + service::{make_service_fn, service_fn}, + Body, Request, Response, Server, StatusCode, + }, + program_transformers::error::ProgramTransformerError, + prometheus::{ + HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder, + }, + std::{net::SocketAddr, sync::Once}, + tracing::{error, info}, +}; + +lazy_static::lazy_static! { + static ref REGISTRY: Registry = Registry::new(); + + static ref VERSION_INFO_METRIC: IntCounterVec = IntCounterVec::new( + Opts::new("version_info", "Plugin version info"), + &["buildts", "git", "package", "proto", "rustc", "solana", "version"] + ).unwrap(); + + static ref REDIS_STREAM_LENGTH: IntGaugeVec = IntGaugeVec::new( + Opts::new("redis_stream_length", "Length of stream in Redis"), + &["stream"] + ).unwrap(); + + static ref REDIS_XADD_STATUS_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xadd_status_count", "Status of messages sent to Redis stream"), + &["stream", "label", "status"] + ).unwrap(); + + static ref REDIS_XREAD_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xread_count", "Count of messages seen"), + &["stream", "consumer"] + ).unwrap(); + + static ref REDIS_XACK_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("redis_xack_count", "Total number of processed messages"), + &["stream", "consumer"] + ).unwrap(); + + static ref PGPOOL_CONNECTIONS: IntGaugeVec = IntGaugeVec::new( + Opts::new("pgpool_connections", "Total number of connections in Postgres Pool"), + &["kind"] + ).unwrap(); + + static ref PROGRAM_TRANSFORMER_TASK_STATUS_COUNT: IntCounterVec = IntCounterVec::new( + Opts::new("program_transformer_task_status_count", "Status of processed messages"), + &["stream", "consumer", "status"], + ).unwrap(); + + static ref INGEST_JOB_TIME: HistogramVec = HistogramVec::new( + HistogramOpts::new("ingest_job_time", "Time taken for ingest jobs"), + &["stream", "consumer"] + ).unwrap(); + + static ref DOWNLOAD_METADATA_FETCHED_COUNT: IntGaugeVec = IntGaugeVec::new( + Opts::new("download_metadata_fetched_count", "Status of download metadata task"), + &["status"] + ).unwrap(); + + static ref INGEST_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("ingest_tasks", "Number of tasks spawned for ingest"), + &["stream", "consumer"] + ).unwrap(); + + static ref ACK_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("ack_tasks", "Number of tasks spawned for ack redis messages"), + &["stream", "consumer"] + ).unwrap(); + + static ref GRPC_TASKS: IntGaugeVec = IntGaugeVec::new( + Opts::new("grpc_tasks", "Number of tasks spawned for writing grpc messages to redis "), + &["label","stream"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_TOTAL_LEAVES: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_total_leaves", "Total number of leaves in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_INCORRECT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_incorrect_proofs", "Number of incorrect proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_NOT_FOUND_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_not_found_proofs", "Number of not found proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_CORRECT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_correct_proofs", "Number of correct proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); + + static ref BUBBLEGUM_TREE_CORRUPT_PROOFS: IntGaugeVec = IntGaugeVec::new( + Opts::new("bubblegum_tree_corrupt_proofs", "Number of corrupt proofs in the bubblegum tree"), + &["tree"] + ).unwrap(); +} + +pub fn run_server(address: SocketAddr) -> anyhow::Result<()> { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { + macro_rules! register { + ($collector:ident) => { + REGISTRY + .register(Box::new($collector.clone())) + .expect("collector can't be registered"); + }; + } + + register!(VERSION_INFO_METRIC); + register!(REDIS_STREAM_LENGTH); + register!(REDIS_XADD_STATUS_COUNT); + register!(REDIS_XREAD_COUNT); + register!(REDIS_XACK_COUNT); + register!(PGPOOL_CONNECTIONS); + register!(PROGRAM_TRANSFORMER_TASK_STATUS_COUNT); + register!(INGEST_JOB_TIME); + register!(DOWNLOAD_METADATA_FETCHED_COUNT); + register!(INGEST_TASKS); + register!(ACK_TASKS); + register!(GRPC_TASKS); + register!(BUBBLEGUM_TREE_TOTAL_LEAVES); + register!(BUBBLEGUM_TREE_INCORRECT_PROOFS); + register!(BUBBLEGUM_TREE_NOT_FOUND_PROOFS); + register!(BUBBLEGUM_TREE_CORRECT_PROOFS); + register!(BUBBLEGUM_TREE_CORRUPT_PROOFS); + + VERSION_INFO_METRIC + .with_label_values(&[ + VERSION_INFO.buildts, + VERSION_INFO.git, + VERSION_INFO.package, + VERSION_INFO.proto, + VERSION_INFO.rustc, + VERSION_INFO.solana, + VERSION_INFO.version, + ]) + .inc(); + }); + + let make_service = make_service_fn(move |_: &AddrStream| async move { + Ok::<_, hyper::Error>(service_fn(move |req: Request| async move { + let response = match req.uri().path() { + "/metrics" => metrics_handler(), + _ => not_found_handler(), + }; + Ok::<_, hyper::Error>(response) + })) + }); + + let server = Server::try_bind(&address)?.serve(make_service); + info!("prometheus server started: http://{address:?}/metrics"); + + tokio::spawn(async move { + if let Err(error) = server.await { + error!("prometheus server failed: {error:?}"); + } + }); + + Ok(()) +} + +fn metrics_handler() -> Response { + let metrics = TextEncoder::new() + .encode_to_string(®ISTRY.gather()) + .unwrap_or_else(|error| { + error!("could not encode custom metrics: {}", error); + String::new() + }); + Response::builder() + .header("content-type", "text/plain") + .body(Body::from(metrics)) + .unwrap() +} + +fn not_found_handler() -> Response { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::empty()) + .unwrap() +} + +pub fn redis_xlen_set(stream: &str, len: usize) { + REDIS_STREAM_LENGTH + .with_label_values(&[stream]) + .set(len as i64); +} + +pub fn ingest_job_time_set(stream: &str, consumer: &str, value: f64) { + INGEST_JOB_TIME + .with_label_values(&[stream, consumer]) + .observe(value); +} + +pub fn redis_xadd_status_inc(stream: &str, label: &str, status: Result<(), ()>, delta: usize) { + REDIS_XADD_STATUS_COUNT + .with_label_values(&[ + stream, + label, + if status.is_ok() { "success" } else { "failed" }, + ]) + .inc_by(delta as u64); +} + +pub fn redis_xread_inc(stream: &str, consumer: &str, delta: usize) { + REDIS_XREAD_COUNT + .with_label_values(&[stream, consumer]) + .inc_by(delta as u64) +} + +pub fn redis_xack_inc(stream: &str, consumer: &str, delta: usize) { + REDIS_XACK_COUNT + .with_label_values(&[stream, consumer]) + .inc_by(delta as u64) +} + +#[derive(Debug, Clone, Copy)] +pub enum PgpoolConnectionsKind { + Total, + Idle, +} + +pub fn pgpool_connections_set(kind: PgpoolConnectionsKind, size: usize) { + PGPOOL_CONNECTIONS + .with_label_values(&[match kind { + PgpoolConnectionsKind::Total => "total", + PgpoolConnectionsKind::Idle => "idle", + }]) + .set(size as i64) +} + +pub fn ingest_tasks_total_inc(stream: &str, consumer: &str) { + INGEST_TASKS.with_label_values(&[stream, consumer]).inc() +} + +pub fn ingest_tasks_total_dec(stream: &str, consumer: &str) { + INGEST_TASKS.with_label_values(&[stream, consumer]).dec() +} + +pub fn ack_tasks_total_inc(stream: &str, consumer: &str) { + ACK_TASKS.with_label_values(&[stream, consumer]).inc() +} + +pub fn ack_tasks_total_dec(stream: &str, consumer: &str) { + ACK_TASKS.with_label_values(&[stream, consumer]).dec() +} + +pub fn grpc_tasks_total_inc(label: &str, stream: &str) { + GRPC_TASKS.with_label_values(&[label, stream]).inc() +} + +pub fn grpc_tasks_total_dec(label: &str, stream: &str) { + GRPC_TASKS.with_label_values(&[label, stream]).dec() +} + +pub fn download_metadata_json_task_status_count_inc(status: u16) { + DOWNLOAD_METADATA_FETCHED_COUNT + .with_label_values(&[&status.to_string()]) + .inc(); +} + +#[derive(Debug, Clone, Copy)] +pub enum ProgramTransformerTaskStatusKind { + Success, + NotImplemented, + DeserializationError, + ParsingError, + ChangeLogEventMalformed, + StorageWriteError, + SerializatonError, + DatabaseError, + AssetIndexError, + DownloadMetadataNotify, + DownloadMetadataSeaOrmError, + DownloadMetadataFetchError, + DownloadMetadataAssetNotFound, + RedisMessageDeserializeError, +} + +impl From for ProgramTransformerTaskStatusKind { + fn from(error: ProgramTransformerError) -> Self { + match error { + ProgramTransformerError::ChangeLogEventMalformed => { + ProgramTransformerTaskStatusKind::ChangeLogEventMalformed + } + ProgramTransformerError::StorageWriteError(_) => { + ProgramTransformerTaskStatusKind::StorageWriteError + } + ProgramTransformerError::NotImplemented => { + ProgramTransformerTaskStatusKind::NotImplemented + } + ProgramTransformerError::DeserializationError(_) => { + ProgramTransformerTaskStatusKind::DeserializationError + } + ProgramTransformerError::SerializatonError(_) => { + ProgramTransformerTaskStatusKind::SerializatonError + } + ProgramTransformerError::ParsingError(_) => { + ProgramTransformerTaskStatusKind::ParsingError + } + ProgramTransformerError::DatabaseError(_) => { + ProgramTransformerTaskStatusKind::DatabaseError + } + ProgramTransformerError::AssetIndexError(_) => { + ProgramTransformerTaskStatusKind::AssetIndexError + } + ProgramTransformerError::DownloadMetadataNotify(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataNotify + } + } + } +} + +impl From for ProgramTransformerTaskStatusKind { + fn from(error: MetadataJsonTaskError) -> Self { + match error { + MetadataJsonTaskError::SeaOrm(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataSeaOrmError + } + MetadataJsonTaskError::Fetch(_) => { + ProgramTransformerTaskStatusKind::DownloadMetadataFetchError + } + MetadataJsonTaskError::AssetNotFound => { + ProgramTransformerTaskStatusKind::DownloadMetadataAssetNotFound + } + } + } +} + +impl From for ProgramTransformerTaskStatusKind { + fn from(_: RedisStreamMessageError) -> Self { + ProgramTransformerTaskStatusKind::RedisMessageDeserializeError + } +} +impl ProgramTransformerTaskStatusKind { + pub const fn to_str(self) -> &'static str { + match self { + ProgramTransformerTaskStatusKind::Success => "success", + ProgramTransformerTaskStatusKind::NotImplemented => "not_implemented", + ProgramTransformerTaskStatusKind::DeserializationError => "deserialization_error", + ProgramTransformerTaskStatusKind::ParsingError => "parsing_error", + ProgramTransformerTaskStatusKind::ChangeLogEventMalformed => { + "changelog_event_malformed" + } + ProgramTransformerTaskStatusKind::StorageWriteError => "storage_write_error", + ProgramTransformerTaskStatusKind::SerializatonError => "serialization_error", + ProgramTransformerTaskStatusKind::DatabaseError => "database_error", + ProgramTransformerTaskStatusKind::AssetIndexError => "asset_index_error", + ProgramTransformerTaskStatusKind::DownloadMetadataNotify => "download_metadata_notify", + ProgramTransformerTaskStatusKind::DownloadMetadataSeaOrmError => { + "download_metadata_sea_orm_error" + } + ProgramTransformerTaskStatusKind::DownloadMetadataFetchError => { + "download_metadata_fetch_error" + } + ProgramTransformerTaskStatusKind::DownloadMetadataAssetNotFound => { + "download_metadata_asset_not_found" + } + ProgramTransformerTaskStatusKind::RedisMessageDeserializeError => { + "redis_message_deserialize_error" + } + } + } +} + +pub fn program_transformer_task_status_inc( + stream: &str, + consumer: &str, + kind: ProgramTransformerTaskStatusKind, +) { + PROGRAM_TRANSFORMER_TASK_STATUS_COUNT + .with_label_values(&[stream, consumer, kind.to_str()]) + .inc() +} + +pub fn update_tree_proof_report(report: &ProofReport) { + BUBBLEGUM_TREE_TOTAL_LEAVES + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.total_leaves as i64); + + BUBBLEGUM_TREE_INCORRECT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.incorrect_proofs as i64); + + BUBBLEGUM_TREE_NOT_FOUND_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.not_found_proofs as i64); + + BUBBLEGUM_TREE_CORRECT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.correct_proofs as i64); + + BUBBLEGUM_TREE_CORRUPT_PROOFS + .with_label_values(&[&report.tree_pubkey.to_string()]) + .set(report.corrupt_proofs as i64); +} diff --git a/grpc-ingest/src/redis.rs b/grpc-ingest/src/redis.rs new file mode 100644 index 000000000..0845f891d --- /dev/null +++ b/grpc-ingest/src/redis.rs @@ -0,0 +1,754 @@ +use { + crate::{ + config::{ConfigIngestStream, REDIS_STREAM_DATA_KEY}, + prom::{ + ack_tasks_total_dec, ack_tasks_total_inc, download_metadata_json_task_status_count_inc, + ingest_job_time_set, ingest_tasks_total_dec, ingest_tasks_total_inc, + program_transformer_task_status_inc, redis_xack_inc, redis_xlen_set, redis_xread_inc, + ProgramTransformerTaskStatusKind, + }, + }, + das_core::{ + DownloadMetadata, DownloadMetadataInfo, DownloadMetadataJsonRetryConfig, + FetchMetadataJsonError, MetadataJsonTaskError, StatusCode, + }, + futures::future::BoxFuture, + program_transformers::{AccountInfo, ProgramTransformer, TransactionInfo}, + redis::{ + aio::MultiplexedConnection, + streams::{StreamId, StreamKey, StreamMaxlen, StreamReadOptions, StreamReadReply}, + AsyncCommands, ErrorKind as RedisErrorKind, RedisResult, Value as RedisValue, + }, + solana_sdk::{pubkey::Pubkey, signature::Signature}, + std::{collections::HashMap, marker::PhantomData, sync::Arc}, + tokio::{ + task::JoinSet, + time::{sleep, Duration}, + }, + tracing::{debug, error, warn}, + yellowstone_grpc_proto::{ + convert_from::{ + create_message_instructions, create_meta_inner_instructions, create_pubkey_vec, + }, + prelude::{SubscribeUpdateAccount, SubscribeUpdateTransaction}, + prost::Message, + }, +}; + +#[derive(thiserror::Error, Debug)] +pub enum RedisStreamMessageError { + #[error("failed to get data (key: {0}) from stream")] + MissingData(String), + #[error("invalid data (key: {0}) from stream")] + InvalidData(String), + #[error("failed to decode message")] + Decode(#[from] yellowstone_grpc_proto::prost::DecodeError), + #[error("received invalid SubscribeUpdateAccount")] + InvalidSubscribeUpdateAccount, + #[error("failed to convert pubkey")] + PubkeyConversion(#[from] std::array::TryFromSliceError), + #[error("JSON deserialization error: {0}")] + JsonDeserialization(#[from] serde_json::Error), +} + +pub trait RedisStreamMessage { + fn try_parse_msg(msg: HashMap) -> Result; + + fn get_data_as_vec( + msg: &HashMap, + ) -> Result<&Vec, RedisStreamMessageError> { + let data = msg.get(REDIS_STREAM_DATA_KEY).ok_or_else(|| { + RedisStreamMessageError::MissingData(REDIS_STREAM_DATA_KEY.to_string()) + })?; + + match data { + RedisValue::Data(data) => Ok(data), + _ => Err(RedisStreamMessageError::InvalidData( + REDIS_STREAM_DATA_KEY.to_string(), + )), + } + } +} + +impl RedisStreamMessage for AccountInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let account_data = Self::get_data_as_vec(&msg)?; + + let SubscribeUpdateAccount { account, slot, .. } = Message::decode(account_data.as_ref())?; + + let account = account.ok_or(RedisStreamMessageError::InvalidSubscribeUpdateAccount)?; + + Ok(Self { + slot, + pubkey: Pubkey::try_from(account.pubkey.as_slice())?, + owner: Pubkey::try_from(account.owner.as_slice())?, + data: account.data, + }) + } +} + +impl RedisStreamMessage for TransactionInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let transaction_data = Self::get_data_as_vec(&msg)?; + + let SubscribeUpdateTransaction { transaction, slot } = + Message::decode(transaction_data.as_ref())?; + + let transaction = transaction.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid SubscribeUpdateTransaction".to_string(), + ) + })?; + let tx = transaction.transaction.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid transaction in SubscribeUpdateTransaction".to_string(), + ) + })?; + let message = tx.message.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid message in SubscribeUpdateTransaction".to_string(), + ) + })?; + let meta = transaction.meta.ok_or_else(|| { + RedisStreamMessageError::InvalidData( + "received invalid meta in SubscribeUpdateTransaction".to_string(), + ) + })?; + + let mut account_keys = create_pubkey_vec(message.account_keys).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })?; + for pubkey in create_pubkey_vec(meta.loaded_writable_addresses).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })? { + account_keys.push(pubkey); + } + for pubkey in create_pubkey_vec(meta.loaded_readonly_addresses).map_err(|e| { + RedisStreamMessageError::Decode(yellowstone_grpc_proto::prost::DecodeError::new(e)) + })? { + account_keys.push(pubkey); + } + + Ok(Self { + slot, + signature: Signature::try_from(transaction.signature.as_slice())?, + account_keys, + message_instructions: create_message_instructions(message.instructions).map_err( + |e| { + RedisStreamMessageError::Decode( + yellowstone_grpc_proto::prost::DecodeError::new(e), + ) + }, + )?, + meta_inner_instructions: create_meta_inner_instructions(meta.inner_instructions) + .map_err(|e| { + RedisStreamMessageError::Decode( + yellowstone_grpc_proto::prost::DecodeError::new(e), + ) + })?, + }) + } +} + +impl RedisStreamMessage for DownloadMetadataInfo { + fn try_parse_msg(msg: HashMap) -> Result { + let metadata_data = Self::get_data_as_vec(&msg)?; + + let info: DownloadMetadataInfo = serde_json::from_slice(metadata_data.as_ref())?; + + Ok(info) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum IngestMessageError { + #[error("Redis stream message parse error: {0}")] + RedisStreamMessage(#[from] RedisStreamMessageError), + #[error("Program transformer error: {0}")] + ProgramTransformer(#[from] program_transformers::error::ProgramTransformerError), + #[error("Download metadata JSON task error: {0}")] + DownloadMetadataJson(#[from] das_core::MetadataJsonTaskError), +} + +pub struct IngestStreamStop { + shutdown_tx: tokio::sync::oneshot::Sender<()>, + control: tokio::task::JoinHandle<()>, +} + +impl IngestStreamStop { + pub async fn stop(self) -> anyhow::Result<()> { + self.shutdown_tx + .send(()) + .map_err(|_| anyhow::anyhow!("Failed to send shutdown signal"))?; + + self.control.await?; + + Ok(()) + } +} + +pub trait MessageHandler: Send + Sync + Clone + 'static { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>>; +} + +pub struct DownloadMetadataJsonHandle(Arc, Arc); + +impl MessageHandler for DownloadMetadataJsonHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let download_metadata = Arc::clone(&self.0); + let download_config = Arc::clone(&self.1); + + Box::pin(async move { + let info = DownloadMetadataInfo::try_parse_msg(input)?; + let response = download_metadata + .handle_download(&info, download_config) + .await; + let status = + if let Err(MetadataJsonTaskError::Fetch(FetchMetadataJsonError::Response { + status: StatusCode::Code(code), + .. + })) = response + { + code.as_u16() + } else { + 200 + }; + + download_metadata_json_task_status_count_inc(status); + + response.map_err(IngestMessageError::DownloadMetadataJson) + }) + } +} + +impl DownloadMetadataJsonHandle { + pub const fn new( + download_metadata: Arc, + config: Arc, + ) -> Self { + Self(download_metadata, config) + } +} + +impl Clone for DownloadMetadataJsonHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0), Arc::clone(&self.1)) + } +} + +pub struct AccountHandle(Arc); + +impl AccountHandle { + pub const fn new(program_transformer: Arc) -> Self { + Self(program_transformer) + } +} + +impl MessageHandler for AccountHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let program_transformer = Arc::clone(&self.0); + Box::pin(async move { + let account = AccountInfo::try_parse_msg(input)?; + program_transformer + .handle_account_update(&account) + .await + .map_err(IngestMessageError::ProgramTransformer) + }) + } +} + +impl Clone for AccountHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +pub struct TransactionHandle(Arc); + +impl TransactionHandle { + pub const fn new(program_transformer: Arc) -> Self { + Self(program_transformer) + } +} + +impl MessageHandler for TransactionHandle { + fn handle( + &self, + input: HashMap, + ) -> BoxFuture<'static, Result<(), IngestMessageError>> { + let program_transformer = Arc::clone(&self.0); + + Box::pin(async move { + let transaction = TransactionInfo::try_parse_msg(input)?; + program_transformer + .handle_transaction(&transaction) + .await + .map_err(IngestMessageError::ProgramTransformer) + }) + } +} + +impl Clone for TransactionHandle { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +#[derive(Clone)] +pub struct Acknowledge { + config: Arc, + connection: MultiplexedConnection, +} + +impl Acknowledge { + pub const fn new(config: Arc, connection: MultiplexedConnection) -> Self { + Self { config, connection } + } +} + +impl Acknowledge { + async fn handle(&self, ids: Vec) { + let mut connection = self.connection.clone(); + let config = &self.config; + + let count = ids.len(); + + match redis::pipe() + .atomic() + .xack(&config.name, &config.group, &ids) + .xdel(&config.name, &ids) + .query_async::<_, redis::Value>(&mut connection) + .await + { + Ok(response) => { + debug!( + target: "acknowledge_handler", + "action=acknowledge_and_delete stream={} response={:?} expected={:?}", + config.name, response, count + ); + + redis_xack_inc(&config.name, &config.consumer, count); + } + Err(e) => { + error!( + target: "acknowledge_handler", + "action=acknowledge_and_delete_failed stream={} error={:?}", + config.name, e + ); + } + } + + ack_tasks_total_dec(&config.name, &config.consumer); + } +} + +pub struct IngestStream { + config: Arc, + connection: Option, + handler: Option, + _handler: PhantomData, +} + +impl IngestStream { + pub fn build() -> Self { + Self { + config: Arc::new(ConfigIngestStream::default()), + connection: None, + handler: None, + _handler: PhantomData, + } + } + + pub fn handler(mut self, handler: H) -> Self { + self.handler = Some(handler); + self + } + + pub fn config(mut self, config: ConfigIngestStream) -> Self { + self.config = Arc::new(config); + self + } + + pub fn connection(mut self, connection: MultiplexedConnection) -> Self { + self.connection = Some(connection); + self + } + + async fn read(&self, connection: &mut MultiplexedConnection) -> RedisResult { + let config = &self.config; + + let opts = StreamReadOptions::default() + .group(&config.group, &config.consumer) + .count(config.batch_size) + .block(250); + + connection + .xread_options(&[&config.name], &[">"], &opts) + .await + } + + pub async fn start(mut self) -> anyhow::Result { + let config = Arc::clone(&self.config); + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + + let mut connection = self.connection.take().expect("Connection is required"); + let handler = self.handler.take().expect("Handler is required"); + + xgroup_create(&mut connection, &config.name, &config.group).await?; + + xgroup_delete_consumer( + &mut connection, + &config.name, + &config.group, + &config.consumer, + ) + .await?; + + xgroup_create_consumer( + &mut connection, + &config.name, + &config.group, + &config.consumer, + ) + .await?; + + let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::(config.xack_buffer_size); + let (ack_shutdown_tx, mut ack_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + + let (msg_tx, mut msg_rx) = + tokio::sync::mpsc::channel::>(config.message_buffer_size); + let (msg_shutdown_tx, mut msg_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + + let config_messages = Arc::clone(&config); + + let messages = tokio::spawn(async move { + let mut tasks = JoinSet::new(); + let config = Arc::clone(&config_messages); + let handler = handler.clone(); + + loop { + tokio::select! { + Some(ids) = msg_rx.recv() => { + for StreamId { id, map } in ids { + if tasks.len() >= config.max_concurrency { + tasks.join_next().await; + } + + let handler = handler.clone(); + let ack_tx = ack_tx.clone(); + let config = Arc::clone(&config); + + ingest_tasks_total_inc(&config.name, &config.consumer); + + tasks.spawn(async move { + let start_time = tokio::time::Instant::now(); + let result = handler.handle(map).await.map_err(IngestMessageError::into); + let elapsed_time = start_time.elapsed().as_secs_f64(); + + ingest_job_time_set(&config.name, &config.consumer, elapsed_time); + + match result { + Ok(()) => { + program_transformer_task_status_inc(&config.name, &config.consumer, ProgramTransformerTaskStatusKind::Success); + } + Err(IngestMessageError::RedisStreamMessage(e)) => { + error!("Failed to process message: {:?}", e); + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); + } + Err(IngestMessageError::DownloadMetadataJson(e)) => { + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); + } + Err(IngestMessageError::ProgramTransformer(e)) => { + error!("Failed to process message: {:?}", e); + program_transformer_task_status_inc(&config.name, &config.consumer, e.into()); + } + } + + if let Err(e) = ack_tx.send(id).await { + error!(target: "ingest_stream", "action=send_ack stream={} error={:?}", &config.name, e); + } + + ingest_tasks_total_dec(&config.name, &config.consumer); + }); + } + } + _ = &mut msg_shutdown_rx => { + break; + } + } + } + + while (tasks.join_next().await).is_some() {} + }); + + let ack = tokio::spawn({ + let config = Arc::clone(&config); + let mut pending = Vec::new(); + let mut tasks = JoinSet::new(); + let handler = Arc::new(Acknowledge::new(Arc::clone(&config), connection.clone())); + + async move { + let deadline = tokio::time::sleep(config.xack_batch_max_idle); + tokio::pin!(deadline); + + loop { + tokio::select! { + Some(id) = ack_rx.recv() => { + pending.push(id); + + if pending.len() >= config.xack_batch_max_size { + if tasks.len() >= config.ack_concurrency { + tasks.join_next().await; + } + + let ids = std::mem::take(&mut pending); + let handler = Arc::clone(&handler); + + + ack_tasks_total_inc(&config.name, &config.consumer); + + tasks.spawn(async move { + handler.handle(ids).await; + }); + + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); + } + } + _ = &mut deadline, if !pending.is_empty() => { + if tasks.len() >= config.ack_concurrency { + tasks.join_next().await; + } + let ids = std::mem::take(&mut pending); + let handler = Arc::clone(&handler); + + ack_tasks_total_inc(&config.name, &config.consumer); + + tasks.spawn(async move { + handler.handle(ids).await; + }); + + deadline.as_mut().reset(tokio::time::Instant::now() + config.xack_batch_max_idle); + } + _ = &mut ack_shutdown_rx => { + break; + } + } + } + + if !pending.is_empty() { + let handler = Arc::clone(&handler); + handler.handle(std::mem::take(&mut pending)).await; + } + + while (tasks.join_next().await).is_some() {} + } + }); + + let labels = vec![config.name.clone()]; + tokio::spawn({ + let connection = connection.clone(); + let config = Arc::clone(&config); + + async move { + let config = Arc::clone(&config); + + loop { + let connection = connection.clone(); + let labels = labels.clone(); + + if let Err(e) = report_xlen(connection, labels).await { + error!(target: "ingest_stream", "action=report_xlen stream={} error={:?}", &config.name, e); + } + + sleep(Duration::from_millis(100)).await; + } + } + }); + + let control = tokio::spawn({ + let mut connection = connection.clone(); + + async move { + let config = Arc::clone(&config); + + debug!(target: "ingest_stream", "action=read_stream_start stream={}", config.name); + + loop { + let config = Arc::clone(&config); + + tokio::select! { + _ = &mut shutdown_rx => { + if let Err(e) = msg_shutdown_tx.send(()) { + error!(target: "ingest_stream", "action=msg_shutdown stream={} error={:?}", &config.name, e); + } + + if let Err(e) = messages.await { + error!(target: "ingest_stream", "action=await_messages stream={} error={:?}", &config.name, e); + } + + if let Err(e) = ack_shutdown_tx.send(()) { + error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", &config.name, e); + } + + if let Err(e) = ack.await { + error!(target: "ingest_stream", "action=ack_shutdown stream={} error={:?}", &config.name, e); + } + + break; + }, + result = self.read(&mut connection) => { + match result { + Ok(reply) => { + for StreamKey { key: _, ids } in reply.keys { + let config = Arc::clone(&config); + let count = ids.len(); + debug!(target: "ingest_stream", "action=xread stream={} count={:?}", &config.name, count); + + redis_xread_inc(&config.name, &config.consumer, count); + + if let Err(e) = msg_tx.send(ids).await { + error!(target: "ingest_stream", "action=send_ids stream={} error={:?}", &config.name, e); + } + } + } + Err(err) => { + error!(target: "ingest_stream", "action=xread stream={} error={:?}", &config.name, err); + } + } + } + } + } + + warn!(target: "ingest_stream", "action=stream_shutdown stream={} stream shutdown", config.name); + } + }); + + Ok(IngestStreamStop { + control, + shutdown_tx, + }) + } +} + +#[derive(Clone)] +pub struct TrackedPipeline { + pipeline: redis::Pipeline, + count: usize, +} + +impl Default for TrackedPipeline { + fn default() -> Self { + Self { + pipeline: redis::pipe(), + count: 0, + } + } +} + +impl TrackedPipeline { + pub fn xadd_maxlen(&mut self, key: &str, maxlen: StreamMaxlen, id: F, field: V) + where + F: redis::ToRedisArgs, + V: redis::ToRedisArgs, + { + self.pipeline + .xadd_maxlen(key, maxlen, id, &[(REDIS_STREAM_DATA_KEY, field)]); + self.count += 1; + } + + pub async fn flush(&mut self, connection: &mut MultiplexedConnection) -> Result { + let result: RedisResult = self.pipeline.atomic().query_async(connection).await; + let count = self.count; + self.count = 0; + self.pipeline.clear(); + + match result { + Ok(_) => Ok(count), + Err(_) => Err(count), + } + } +} + +pub async fn report_xlen( + mut connection: C, + streams: Vec, +) -> anyhow::Result<()> { + let mut pipe = redis::pipe(); + for stream in &streams { + pipe.xlen(stream); + } + let xlens: Vec = pipe.query_async(&mut connection).await?; + + for (stream, xlen) in streams.iter().zip(xlens.into_iter()) { + redis_xlen_set(stream, xlen); + } + + Ok(()) +} + +pub async fn xgroup_create( + connection: &mut C, + name: &str, + group: &str, +) -> anyhow::Result<()> { + let result: RedisResult = connection.xgroup_create_mkstream(name, group, "0").await; + if let Err(error) = result { + if !(error.kind() == RedisErrorKind::ExtensionError + && error.detail() == Some("Consumer Group name already exists") + && error.code() == Some("BUSYGROUP")) + { + return Err(error.into()); + } + } + + Ok(()) +} + +pub async fn xgroup_create_consumer( + connection: &mut C, + name: &str, + group: &str, + consumer: &str, +) -> anyhow::Result<()> { + let result: RedisResult = redis::cmd("XGROUP") + .arg("CREATECONSUMER") + .arg(name) + .arg(group) + .arg(consumer) + .query_async(connection) + .await; + + match result { + Ok(_) => Ok(()), + Err(error) => Err(error.into()), + } +} + +pub async fn xgroup_delete_consumer( + connection: &mut C, + name: &str, + group: &str, + consumer: &str, +) -> anyhow::Result<()> { + let result: RedisResult = redis::cmd("XGROUP") + .arg("DELCONSUMER") + .arg(name) + .arg(group) + .arg(consumer) + .query_async(connection) + .await; + + match result { + Ok(_) => Ok(()), + Err(error) => Err(error.into()), + } +} diff --git a/grpc-ingest/src/tracing.rs b/grpc-ingest/src/tracing.rs new file mode 100644 index 000000000..2d50f785c --- /dev/null +++ b/grpc-ingest/src/tracing.rs @@ -0,0 +1,33 @@ +use { + opentelemetry_sdk::trace::{self, Sampler}, + std::env, + tracing_subscriber::{filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}, +}; + +pub fn init() -> anyhow::Result<()> { + let open_tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_service_name(env::var("CARGO_PKG_NAME")?) + .with_auto_split_batch(true) + .with_trace_config(trace::config().with_sampler(Sampler::TraceIdRatioBased(0.25))) + .install_batch(opentelemetry_sdk::runtime::Tokio)?; + let jeager_layer = tracing_opentelemetry::layer().with_tracer(open_tracer); + + let env_filter = EnvFilter::builder() + .parse(env::var(EnvFilter::DEFAULT_ENV).unwrap_or_else(|_| "info,sqlx=warn".to_owned()))?; + + let is_atty = atty::is(atty::Stream::Stdout) && atty::is(atty::Stream::Stderr); + let io_layer = tracing_subscriber::fmt::layer().with_ansi(is_atty); + + let registry = tracing_subscriber::registry() + .with(jeager_layer) + .with(env_filter) + .with(io_layer); + + if env::var_os("RUST_LOG_JSON").is_some() { + let json_layer = tracing_subscriber::fmt::layer().json().flatten_event(true); + registry.with(json_layer).try_init() + } else { + registry.try_init() + } + .map_err(Into::into) +} diff --git a/grpc-ingest/src/util.rs b/grpc-ingest/src/util.rs new file mode 100644 index 000000000..0a7800a12 --- /dev/null +++ b/grpc-ingest/src/util.rs @@ -0,0 +1,19 @@ +use { + async_stream::stream, + futures::stream::{BoxStream, StreamExt}, + tokio::signal::unix::{signal, SignalKind}, +}; + +pub fn create_shutdown() -> anyhow::Result> { + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigterm = signal(SignalKind::terminate())?; + Ok(stream! { + loop { + yield tokio::select! { + _ = sigint.recv() => "SIGINT", + _ = sigterm.recv() => "SIGTERM", + }; + } + } + .boxed()) +} diff --git a/grpc-ingest/src/version.rs b/grpc-ingest/src/version.rs new file mode 100644 index 000000000..b9da62845 --- /dev/null +++ b/grpc-ingest/src/version.rs @@ -0,0 +1,22 @@ +use {serde::Serialize, std::env}; + +#[derive(Debug, Serialize)] +pub struct Version { + pub package: &'static str, + pub version: &'static str, + pub proto: &'static str, + pub solana: &'static str, + pub git: &'static str, + pub rustc: &'static str, + pub buildts: &'static str, +} + +pub const VERSION: Version = Version { + package: env!("CARGO_PKG_NAME"), + version: env!("CARGO_PKG_VERSION"), + proto: env!("YELLOWSTONE_GRPC_PROTO_VERSION"), + solana: env!("SOLANA_SDK_VERSION"), + git: env!("GIT_VERSION"), + rustc: env!("VERGEN_RUSTC_SEMVER"), + buildts: env!("VERGEN_BUILD_TIMESTAMP"), +}; diff --git a/migration/Cargo.toml b/migration/Cargo.toml index c0202ffdd..6ce0612be 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -9,7 +9,10 @@ publish = { workspace = true } async-std = { workspace = true, features = ["attributes", "tokio1"] } enum-iterator = { workspace = true } enum-iterator-derive = { workspace = true } -sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres"] } +sea-orm-migration = { workspace = true, features = [ + "runtime-tokio-rustls", + "sqlx-postgres", +] } [lints] workspace = true diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index fd17a2bed..0b8494cd1 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -14,6 +14,7 @@ cadence = { workspace = true } cadence-macros = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive", "cargo"] } +das-core = { workspace = true } digital_asset_types = { workspace = true, features = [ "json_types", "sql_types", diff --git a/nft_ingester/src/backfiller.rs b/nft_ingester/src/backfiller.rs index a8a4f1685..39d8106c2 100644 --- a/nft_ingester/src/backfiller.rs +++ b/nft_ingester/src/backfiller.rs @@ -66,6 +66,7 @@ const BLOCK_CACHE_SIZE: usize = 300_000; const MAX_CACHE_COST: i64 = 32; const BLOCK_CACHE_DURATION: u64 = 172800; +#[allow(dead_code)] struct SlotSeq(u64, u64); /// Main public entry point for backfiller task. pub fn setup_backfiller( @@ -799,11 +800,7 @@ impl<'a, T: Messenger> Backfiller<'a, T> { .build(DbBackend::Postgres); let start_seq_vec = MaxSeqItem::find_by_statement(query).all(&self.db).await?; - let start_seq = if let Some(seq) = start_seq_vec.last().map(|row| row.seq) { - seq - } else { - 0 - }; + let start_seq = start_seq_vec.last().map(|row| row.seq).unwrap_or_default(); // Get all rows for the tree that have not yet been backfilled. let mut query = backfill_items::Entity::find() diff --git a/nft_ingester/src/plerkle.rs b/nft_ingester/src/plerkle.rs index bef3d4ff6..446f558b1 100644 --- a/nft_ingester/src/plerkle.rs +++ b/nft_ingester/src/plerkle.rs @@ -17,7 +17,7 @@ pub enum PlerkleDeserializerError { pub struct PlerkleAccountInfo<'a>(pub plerkle_serialization::AccountInfo<'a>); -impl<'a> TryFrom> for AccountInfo { +impl TryFrom> for AccountInfo { type Error = PlerkleDeserializerError; fn try_from(value: PlerkleAccountInfo) -> Result { diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs index 17ec935a0..7d52439a9 100644 --- a/nft_ingester/src/tasks/common/mod.rs +++ b/nft_ingester/src/tasks/common/mod.rs @@ -2,10 +2,10 @@ use { super::{BgTask, FromTaskData, IngesterError, IntoTaskData, TaskData}, async_trait::async_trait, chrono::{NaiveDateTime, Utc}, + das_core::{DownloadMetadataInfo, DownloadMetadataNotifier}, digital_asset_types::dao::asset_data, futures::future::BoxFuture, log::debug, - program_transformers::{DownloadMetadataInfo, DownloadMetadataNotifier}, reqwest::{Client, ClientBuilder}, sea_orm::*, serde::{Deserialize, Serialize}, @@ -25,7 +25,7 @@ pub fn create_download_metadata_notifier( 'static, Result<(), Box>, > { - let (asset_data_id, uri) = info.into_inner(); + let (asset_data_id, uri, _slot) = info.into_inner(); let task = DownloadMetadata { asset_data_id, uri, diff --git a/nft_ingester/src/tasks/mod.rs b/nft_ingester/src/tasks/mod.rs index ec7a813fe..6c6f854b6 100644 --- a/nft_ingester/src/tasks/mod.rs +++ b/nft_ingester/src/tasks/mod.rs @@ -324,8 +324,8 @@ impl TaskManager { tokio::task::spawn(async move { while let Some(task) = receiver.recv().await { if let Some(task_created_time) = task.created_at { - let bus_time = - Utc::now().timestamp_millis() - task_created_time.timestamp_millis(); + let bus_time = Utc::now().timestamp_millis() + - task_created_time.and_utc().timestamp_millis(); metric! { statsd_histogram!("ingester.bgtask.bus_time", bus_time as u64, "type" => task.name); } diff --git a/ops/Cargo.toml b/ops/Cargo.toml index e6cb7a870..6a0964ae9 100644 --- a/ops/Cargo.toml +++ b/ops/Cargo.toml @@ -11,27 +11,32 @@ name = "das-ops" [dependencies] anchor-client = { workspace = true } anyhow = { workspace = true } -backon = { workspace = true } borsh = { workspace = true } +bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } clap = { workspace = true, features = ["derive", "cargo", "env"] } +das-bubblegum = { workspace = true } das-core = { workspace = true } digital_asset_types = { workspace = true } env_logger = { workspace = true } figment = { workspace = true } -flatbuffers = { workspace = true } futures = { workspace = true } indicatif = { workspace = true } log = { workspace = true } mpl-bubblegum = { workspace = true } -plerkle_messenger = { workspace = true } -plerkle_serialization = { workspace = true } +program_transformers = { workspace = true } sea-orm = { workspace = true } solana-account-decoder = { workspace = true } solana-client = { workspace = true } +solana-program = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true } +sqlx = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } +mpl-token-metadata = { workspace = true } +serde_json = { workspace = true } +reqwest = { workspace = true } \ No newline at end of file diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 000000000..d81cc4a24 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,41 @@ +### DAS Ops + +DAS Ops is a collection of operational tools and scripts for managing and maintaining the Digital Asset RPC infrastructure. + +> **Note:** Run these commands from the root of the project + +### Setup + +```bash +sudo docker compose up db +``` + +### Running the cli + +```bash +cargo run --bin das-ops -- --help +``` + +#### Required Args + +- `--solana-rpc-url` - RPC URL of the Solana cluster +- `--database-url` - URL of the Postgres database (if using Docker: `postgres://solana:solana@localhost:5432/solana`) + +### Commands + +- `account` : Account related operations + + #### Subcommands + + - `program ` command is used to backfill the index against on-chain accounts owned by a program + + - `single ` command is used to backfill the index against a single account + + - `nft ` command is used to backfill the index against an NFT mint, token metadata, and token account + +- `bubblegum` : Bubblegum program related operations + + #### Subcommands + + - `backfill` command is used to cross-reference the index against on-chain accounts. It crawls through trees and backfills any missed tree transactions. + - `replay ` command is used to replay the Bubblegum program transactions for a given tree address and parse all the cNFT instructions diff --git a/ops/src/account/account_details.rs b/ops/src/account/account_details.rs deleted file mode 100644 index 0490f1ca1..000000000 --- a/ops/src/account/account_details.rs +++ /dev/null @@ -1,30 +0,0 @@ -use anyhow::Result; -use das_core::Rpc; -use solana_sdk::{account::Account, pubkey::Pubkey}; - -pub struct AccountDetails<'a> { - pub account: Account, - pub slot: u64, - pub pubkey: &'a Pubkey, -} - -impl<'a> AccountDetails<'a> { - pub fn new(account: Account, slot: u64, pubkey: &'a Pubkey) -> Self { - Self { - account, - slot, - pubkey, - } - } - - pub async fn fetch(rpc: &Rpc, pubkey: &'a Pubkey) -> Result { - let account_response = rpc.get_account(pubkey).await?; - let slot = account_response.context.slot; - - let account = account_response - .value - .ok_or_else(|| anyhow::anyhow!("Account not found for pubkey: {}", pubkey))?; - - Ok(Self::new(account, slot, pubkey)) - } -} diff --git a/ops/src/account/account_info.rs b/ops/src/account/account_info.rs new file mode 100644 index 000000000..6ce48a544 --- /dev/null +++ b/ops/src/account/account_info.rs @@ -0,0 +1,28 @@ +use anyhow::Result; +use das_core::Rpc; +use program_transformers::AccountInfo; +use solana_sdk::pubkey::Pubkey; + +#[derive(thiserror::Error, Debug)] +pub enum AccountInfoError { + #[error("account not found for pubkey: {pubkey}")] + NotFound { pubkey: Pubkey }, + #[error("failed to fetch account info")] + SolanaRequestError(#[from] solana_client::client_error::ClientError), +} + +pub async fn fetch(rpc: &Rpc, pubkey: Pubkey) -> Result { + let account_response = rpc.get_account(&pubkey).await?; + let slot = account_response.context.slot; + + let account = account_response + .value + .ok_or(AccountInfoError::NotFound { pubkey })?; + + Ok(AccountInfo { + slot, + pubkey, + owner: account.owner, + data: account.data, + }) +} diff --git a/ops/src/account/cmd.rs b/ops/src/account/cmd.rs index 659758d1c..c36fdd22e 100644 --- a/ops/src/account/cmd.rs +++ b/ops/src/account/cmd.rs @@ -1,4 +1,4 @@ -use super::{program, single}; +use super::{nft, program, single}; use anyhow::Result; use clap::{Args, Subcommand}; @@ -10,6 +10,9 @@ pub enum Commands { /// The 'single' command is used to backfill the index against a single account. #[clap(name = "single")] Single(single::Args), + /// The 'nft' command is used to backfill the index against an NFT mint, token metadata, and token account. + #[clap(name = "nft")] + Nft(nft::Args), } #[derive(Debug, Clone, Args)] @@ -26,6 +29,9 @@ pub async fn subcommand(subcommand: AccountCommand) -> Result<()> { Commands::Single(args) => { single::run(args).await?; } + Commands::Nft(args) => { + nft::run(args).await?; + } } Ok(()) diff --git a/ops/src/account/mod.rs b/ops/src/account/mod.rs index 6eeff799f..e770cd362 100644 --- a/ops/src/account/mod.rs +++ b/ops/src/account/mod.rs @@ -1,5 +1,6 @@ -mod account_details; +mod account_info; mod cmd; +mod nft; mod program; mod single; diff --git a/ops/src/account/nft.rs b/ops/src/account/nft.rs new file mode 100644 index 000000000..1f3d2ad20 --- /dev/null +++ b/ops/src/account/nft.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use anyhow::Result; +use tokio::task::JoinHandle; + +use super::account_info; +use log::error; + +use clap::Parser; +use das_core::{ + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, +}; +use mpl_token_metadata::accounts::Metadata; +use program_transformers::ProgramTransformer; +use solana_sdk::pubkey::Pubkey; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, + + /// NFT Mint address + #[clap(value_parser = parse_pubkey)] + pub mint: Pubkey, +} + +fn parse_pubkey(s: &str) -> Result { + Pubkey::try_from(s).map_err(|_| "Failed to parse public key") +} + +pub async fn run(config: Args) -> Result<()> { + let rpc = Rpc::from_config(&config.solana); + let pool = connect_db(&config.database).await?; + let metadata_json_download_db_pool = pool.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; + + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender.clone()).await; + + let mint = config.mint; + + let metadata = Metadata::find_pda(&mint).0; + + let mut accounts_to_fetch = vec![mint, metadata]; + + let token_account = rpc.get_token_largest_account(mint).await; + + if let Ok(token_account) = token_account { + accounts_to_fetch.push(token_account); + } + + let program_transformer = Arc::new(ProgramTransformer::new(pool, download_metadata_notifier)); + let mut tasks = Vec::new(); + + for account in accounts_to_fetch { + let program_transformer = Arc::clone(&program_transformer); + let rpc = rpc.clone(); + + let task: JoinHandle> = tokio::spawn(async move { + let account_info = account_info::fetch(&rpc, account).await?; + if let Err(e) = program_transformer + .handle_account_update(&account_info) + .await + { + error!("Failed to handle account update: {:?}", e); + } + + Ok(()) + }); + + tasks.push(task); + } + + futures::future::try_join_all(tasks).await?; + + drop(metadata_json_download_sender); + + drop(program_transformer); + + metadata_json_download_worker.await?; + + Ok(()) +} diff --git a/ops/src/account/program.rs b/ops/src/account/program.rs index 7dd990582..590ef140d 100644 --- a/ops/src/account/program.rs +++ b/ops/src/account/program.rs @@ -1,24 +1,20 @@ +use super::account_info; use anyhow::Result; - -use super::account_details::AccountDetails; use clap::Parser; -use das_core::{MetricsArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs}; -use flatbuffers::FlatBufferBuilder; -use plerkle_serialization::{ - serializer::serialize_account, solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2, +use das_core::{ + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, }; +use futures::{stream::FuturesUnordered, StreamExt}; +use log::error; +use program_transformers::{AccountInfo, ProgramTransformer}; use solana_sdk::pubkey::Pubkey; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::task; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Redis configuration - #[clap(flatten)] - pub queue: QueueArgs, - - /// Metrics configuration - #[clap(flatten)] - pub metrics: MetricsArgs, - /// Solana configuration #[clap(flatten)] pub solana: SolanaRpcArgs, @@ -30,6 +26,22 @@ pub struct Args { /// The public key of the program to backfill #[clap(value_parser = parse_pubkey)] pub program: Pubkey, + + /// The maximum buffer size for accounts + #[arg(long, env, default_value = "10000")] + pub max_buffer_size: usize, + + /// The number of worker threads + #[arg(long, env, default_value = "1000")] + pub account_worker_count: usize, + + /// Metadata JSON download worker configuration + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, } fn parse_pubkey(s: &str) -> Result { @@ -37,45 +49,70 @@ fn parse_pubkey(s: &str) -> Result { } pub async fn run(config: Args) -> Result<()> { - let rpc = Rpc::from_config(config.solana); - let queue = QueuePool::try_from_config(config.queue).await?; + let rpc = Rpc::from_config(&config.solana); + let pool = connect_db(&config.database).await?; + let num_workers = config.account_worker_count; - let accounts = rpc.get_program_accounts(&config.program, None).await?; + let metadata_json_download_db_pool = pool.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; + let (tx, mut rx) = mpsc::channel::>(config.max_buffer_size); + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender.clone()).await; + + let mut workers = FuturesUnordered::new(); + let program_transformer = Arc::new(ProgramTransformer::new(pool, download_metadata_notifier)); + + let account_info_worker_manager = tokio::spawn(async move { + while let Some(account_infos) = rx.recv().await { + if workers.len() >= num_workers { + workers.next().await; + } + + for account_info in account_infos { + let program_transformer = Arc::clone(&program_transformer); + + let worker = task::spawn(async move { + if let Err(e) = program_transformer + .handle_account_update(&account_info) + .await + { + error!("Failed to handle account update: {:?}", e); + } + }); + + workers.push(worker); + } + } + + while (workers.next().await).is_some() {} + }); + + let accounts = rpc.get_program_accounts(&config.program, None).await?; let accounts_chunks = accounts.chunks(config.batch_size); for batch in accounts_chunks { let results = futures::future::try_join_all( batch .iter() - .map(|(pubkey, _account)| AccountDetails::fetch(&rpc, pubkey)), + .cloned() + .map(|(pubkey, _account)| account_info::fetch(&rpc, pubkey)), ) .await?; - for account_detail in results { - let AccountDetails { - account, - slot, - pubkey, - } = account_detail; - let builder = FlatBufferBuilder::new(); - let account_info = ReplicaAccountInfoV2 { - pubkey: &pubkey.to_bytes(), - lamports: account.lamports, - owner: &account.owner.to_bytes(), - executable: account.executable, - rent_epoch: account.rent_epoch, - data: &account.data, - write_version: 0, - txn_signature: None, - }; - - let fbb = serialize_account(builder, &account_info, slot, false); - let bytes = fbb.finished_data(); - - queue.push_account_backfill(bytes).await?; - } + tx.send(results).await?; } + account_info_worker_manager.await?; + + drop(metadata_json_download_sender); + + metadata_json_download_worker.await?; + Ok(()) } diff --git a/ops/src/account/single.rs b/ops/src/account/single.rs index 41269bb05..d62aa3ed6 100644 --- a/ops/src/account/single.rs +++ b/ops/src/account/single.rs @@ -1,23 +1,24 @@ +use std::sync::Arc; + use anyhow::Result; -use super::account_details::AccountDetails; +use super::account_info; use clap::Parser; -use das_core::{MetricsArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs}; -use flatbuffers::FlatBufferBuilder; -use plerkle_serialization::{ - serializer::serialize_account, solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2, +use das_core::{ + connect_db, create_download_metadata_notifier, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, Rpc, SolanaRpcArgs, }; +use program_transformers::ProgramTransformer; use solana_sdk::pubkey::Pubkey; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Redis configuration + /// Database configuration #[clap(flatten)] - pub queue: QueueArgs, + pub database: PoolArgs, - /// Metrics configuration #[clap(flatten)] - pub metrics: MetricsArgs, + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, /// Solana configuration #[clap(flatten)] @@ -32,30 +33,30 @@ fn parse_pubkey(s: &str) -> Result { } pub async fn run(config: Args) -> Result<()> { - let rpc = Rpc::from_config(config.solana); - let queue = QueuePool::try_from_config(config.queue).await?; - - let AccountDetails { - account, - slot, - pubkey, - } = AccountDetails::fetch(&rpc, &config.account).await?; - let builder = FlatBufferBuilder::new(); - let account_info = ReplicaAccountInfoV2 { - pubkey: &pubkey.to_bytes(), - lamports: account.lamports, - owner: &account.owner.to_bytes(), - executable: account.executable, - rent_epoch: account.rent_epoch, - data: &account.data, - write_version: 0, - txn_signature: None, - }; - - let fbb = serialize_account(builder, &account_info, slot, false); - let bytes = fbb.finished_data(); - - queue.push_account_backfill(bytes).await?; + let rpc = Rpc::from_config(&config.solana); + let pool = connect_db(&config.database).await?; + let metadata_json_download_db_pool = pool.clone(); + + let (metadata_json_download_worker, metadata_json_download_sender) = + config.metadata_json_download_worker.start( + metadata_json_download_db_pool, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + )?; + + { + let download_metadata_notifier = + create_download_metadata_notifier(metadata_json_download_sender).await; + + let program_transformer = ProgramTransformer::new(pool, download_metadata_notifier); + + let account_info = account_info::fetch(&rpc, config.account).await?; + + program_transformer + .handle_account_update(&account_info) + .await?; + } + + metadata_json_download_worker.await?; Ok(()) } diff --git a/ops/src/bubblegum/README.md b/ops/src/bubblegum/README.md index d1dc5772b..20f4bf235 100644 --- a/ops/src/bubblegum/README.md +++ b/ops/src/bubblegum/README.md @@ -8,79 +8,6 @@ Command line arguments can also be set through environment variables. ### Backfill -The `backfill` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL, and the messenger Redis URL. +The `backfill` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL. -**warning**: The command expects full archive access to transactions. Before proceeding ensure your RPC is able to serve complete transaction history for Solana. - -```mermaid -flowchart - start((Start)) -->init[Initialize RPC, DB] - init --> fetchTreesDB[Fetch Trees from DB] - fetchTreesDB --> findGapsDB[Find Gaps in DB] - findGapsDB --> enqueueGapFills[Enqueue Gap Fills] - enqueueGapFills --> gapWorkerManager[Gap Worker Manager] - gapWorkerManager --> crawlSignatures[Crawl Solana RPC] - crawlSignatures --> enqueueSignatures[Enqueue Signatures] - enqueueSignatures --> transactionWorkerManager[Transaction Worker Manager] - transactionWorkerManager --> fetchTransactionsRPC[Fetch Transactions RPC] - fetchTransactionsRPC --> processTransactions[Push Transaction to Messenger] - processTransactions ---> Finished -``` - -``` -Usage: das-ops bubblegum backfill [OPTIONS] --database-url --messenger-redis-url --solana-rpc-url - -Options: - --tree-crawler-count - Number of tree crawler workers [env: TREE_CRAWLER_COUNT=] [default: 20] - --signature-channel-size - The size of the signature channel [env: SIGNATURE_CHANNEL_SIZE=] [default: 10000] - --gap-channel-size - The size of the signature channel [env: GAP_CHANNEL_SIZE=] [default: 1000] - --transaction-worker-count - The number of transaction workers [env: TRANSACTION_WORKER_COUNT=] [default: 100] - --gap-worker-count - The number of gap workers [env: GAP_WORKER_COUNT=] [default: 25] - --only-trees - The list of trees to crawl. If not specified, all trees will be crawled [env: ONLY_TREES=] - --database-url - The database URL [env: DATABASE_URL=] - --database-max-connections - The maximum number of connections to the database [env: DATABASE_MAX_CONNECTIONS=] [default: 125] - --database-min-connections - The minimum number of connections to the database [env: DATABASE_MIN_CONNECTIONS=] [default: 5] - --messenger-redis-url - [env: MESSENGER_REDIS_URL=] - --messenger-redis-batch-size - [env: MESSENGER_REDIS_BATCH_SIZE=] [default: 100] - --messenger-queue-connections - [env: MESSENGER_QUEUE_CONNECTIONS=] [default: 25] - --messenger-queue-stream - [env: MESSENGER_QUEUE_STREAM=] [default: TXNFILL] - --metrics-host - [env: METRICS_HOST=] [default: 127.0.0.1] - --metrics-port - [env: METRICS_PORT=] [default: 8125] - --metrics-prefix - [env: METRICS_PREFIX=] [default: das.backfiller] - --solana-rpc-url - [env: SOLANA_RPC_URL=] - -h, --help - Print help -``` - -### Metrics - -The bubblegum command provides several metrics for monitoring performance and status: - -Metric | Description ---- | --- -transaction.failed | Count of failed transaction -transaction.succeeded | Count of successfully queued transaction -transaction.queued | Time for a transaction to be queued -gap.failed | Count of failed gap crawling -gap.succeeded | Count of successfully crawled gaps -gap.queued | Time for a gap to be queued -tree.succeeded | Count of completed tree crawl -tree.crawled | Time to crawl a tree -job.completed | Time to complete the job +**warning**: The command expects full archive access to transactions. Before proceeding ensure your RPC is able to serve complete transaction history for Solana. \ No newline at end of file diff --git a/ops/src/bubblegum/audit.rs b/ops/src/bubblegum/audit.rs deleted file mode 100644 index 4da0a2652..000000000 --- a/ops/src/bubblegum/audit.rs +++ /dev/null @@ -1,88 +0,0 @@ -use anyhow::Result; -use clap::Parser; -use das_core::{connect_db, MetricsArgs, PoolArgs, Rpc, SolanaRpcArgs}; -use digital_asset_types::dao::cl_audits_v2; -use futures::future; -use sea_orm::{CursorTrait, EntityTrait, SqlxPostgresConnector}; -use solana_sdk::signature::Signature; -use solana_transaction_status::EncodedConfirmedTransactionWithStatusMeta; - -use tokio::io::{stdout, AsyncWriteExt}; - -#[derive(Debug, Parser, Clone)] -pub struct Args { - /// Database configuration - #[clap(flatten)] - pub database: PoolArgs, - - /// Metrics configuration - #[clap(flatten)] - pub metrics: MetricsArgs, - - /// Solana configuration - #[clap(flatten)] - pub solana: SolanaRpcArgs, - - #[arg(long, env, default_value = "10000")] - pub batch_size: u64, -} - -pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(config.database).await?; - - let solana_rpc = Rpc::from_config(config.solana); - - let mut output = stdout(); - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - let mut after = None; - - loop { - let mut query = cl_audits_v2::Entity::find().cursor_by(cl_audits_v2::Column::Id); - let mut query = query.first(config.batch_size); - - if let Some(after) = after { - query = query.after(after); - } - - let entries = query.all(&conn).await?; - - let mut transactions = vec![]; - - for entry in entries.clone() { - transactions.push(fetch_transaction(entry, solana_rpc.clone())); - } - - let transactions = future::join_all(transactions).await; - - for (signature, transaction) in transactions.into_iter().flatten() { - if let Some(meta) = transaction.transaction.meta { - if meta.err.is_some() { - output - .write_all(format!("{}\n", signature).as_bytes()) - .await?; - - output.flush().await?; - } - } - } - - after = entries.last().map(|cl_audit_v2| cl_audit_v2.id); - - if entries.is_empty() { - break; - } - } - - Ok(()) -} - -async fn fetch_transaction( - entry: cl_audits_v2::Model, - solana_rpc: Rpc, -) -> Result<(Signature, EncodedConfirmedTransactionWithStatusMeta)> { - let signature = Signature::try_from(entry.tx.as_ref())?; - - let transaction = solana_rpc.get_transaction(&signature).await?; - - Ok((signature, transaction)) -} diff --git a/ops/src/bubblegum/backfiller.rs b/ops/src/bubblegum/backfiller.rs index 8b0e5ed4f..c13315aaa 100644 --- a/ops/src/bubblegum/backfiller.rs +++ b/ops/src/bubblegum/backfiller.rs @@ -1,325 +1,53 @@ -use super::tree::{TreeErrorKind, TreeGapFill, TreeGapModel, TreeResponse}; use anyhow::Result; -use cadence_macros::{statsd_count, statsd_time}; use clap::Parser; -use das_core::{ - connect_db, setup_metrics, MetricsArgs, PoolArgs, QueueArgs, QueuePool, Rpc, SolanaRpcArgs, -}; -use digital_asset_types::dao::cl_audits_v2; -use flatbuffers::FlatBufferBuilder; -use futures::{stream::FuturesUnordered, StreamExt}; -use indicatif::HumanDuration; -use log::{error, info}; -use plerkle_serialization::serializer::seralize_encoded_transaction_with_status; -use sea_orm::{ - ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QueryOrder, SqlxPostgresConnector, -}; -use solana_sdk::signature::Signature; -use std::time::Instant; -use tokio::{sync::mpsc, task::JoinHandle}; +use das_bubblegum::{start_backfill, BackfillArgs, BubblegumContext}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; #[derive(Debug, Parser, Clone)] pub struct Args { - /// Number of tree crawler workers - #[arg(long, env, default_value = "20")] - pub tree_crawler_count: usize, - - /// The size of the signature channel. - #[arg(long, env, default_value = "10000")] - pub signature_channel_size: usize, - - /// The size of the signature channel. - #[arg(long, env, default_value = "1000")] - pub gap_channel_size: usize, - - /// The number of transaction workers. - #[arg(long, env, default_value = "100")] - pub transaction_worker_count: usize, - - /// The number of gap workers. - #[arg(long, env, default_value = "25")] - pub gap_worker_count: usize, - - /// The list of trees to crawl. If not specified, all trees will be crawled. - #[arg(long, env, use_value_delimiter = true)] - pub only_trees: Option>, + /// Backfill Bubblegum Args + #[clap(flatten)] + pub backfill_bubblegum: BackfillArgs, /// Database configuration #[clap(flatten)] pub database: PoolArgs, - /// Redis configuration - #[clap(flatten)] - pub queue: QueueArgs, - - /// Metrics configuration - #[clap(flatten)] - pub metrics: MetricsArgs, - /// Solana configuration #[clap(flatten)] pub solana: SolanaRpcArgs, } -/// Runs the backfilling process for the tree crawler. +/// Executes the backfilling operation for the tree crawler. /// -/// This function initializes the necessary components for the backfilling process, -/// including database connections, RPC clients, and worker managers for handling -/// transactions and gaps. It then proceeds to fetch the trees that need to be crawled -/// and manages the crawling process across multiple workers. +/// This function initializes the necessary components for the backfilling operation, +/// such as database connections and RPC clients, and then delegates the actual +/// backfilling logic to the `das_bubblegum_backfill` crate. /// -/// The function handles the following major tasks: -/// - Establishing connections to the database and initializing RPC clients. -/// - Setting up channels for communication between different parts of the system. -/// - Spawning worker managers for processing transactions and gaps. -/// - Fetching trees from the database and managing their crawling process. -/// - Reporting metrics and logging information throughout the process. +/// The function undertakes the following key tasks: +/// - Establishes database connections and initializes RPC clients. +/// - Creates a context for the backfilling operation. +/// - Invokes the `start_bubblegum_backfill` function from the `das_bubblegum_backfill` crate. /// /// # Arguments /// -/// * `config` - A configuration object containing settings for the backfilling process, -/// including database, RPC, and worker configurations. +/// * `config` - A configuration object that includes settings for the backfilling operation, +/// such as database, RPC, and worker configurations. /// /// # Returns /// -/// This function returns a `Result` which is `Ok` if the backfilling process completes -/// successfully, or an `Err` with an appropriate error message if any part of the process -/// fails. +/// This function returns a `Result` which is `Ok` if the backfilling operation is completed +/// successfully, or an `Err` with a relevant error message if any part of the operation +/// encounters issues. /// /// # Errors /// -/// This function can return errors related to database connectivity, RPC failures, -/// or issues with spawning and managing worker tasks. +/// Potential errors can arise from database connectivity issues or RPC failures. pub async fn run(config: Args) -> Result<()> { - let pool = connect_db(config.database).await?; - - let solana_rpc = Rpc::from_config(config.solana); - let transaction_solana_rpc = solana_rpc.clone(); - let gap_solana_rpc = solana_rpc.clone(); - - setup_metrics(config.metrics)?; - - let (sig_sender, mut sig_receiver) = mpsc::channel::(config.signature_channel_size); - let gap_sig_sender = sig_sender.clone(); - let (gap_sender, mut gap_receiver) = mpsc::channel::(config.gap_channel_size); - - let queue = QueuePool::try_from_config(config.queue).await?; - - let transaction_worker_count = config.transaction_worker_count; - - let transaction_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - - while let Some(signature) = sig_receiver.recv().await { - if handlers.len() >= transaction_worker_count { - handlers.next().await; - } - - let solana_rpc = transaction_solana_rpc.clone(); - let queue = queue.clone(); - - let handle = spawn_transaction_worker(solana_rpc, queue, signature); - - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - }); - - let gap_worker_count = config.gap_worker_count; - - let gap_worker_manager = tokio::spawn(async move { - let mut handlers = FuturesUnordered::new(); - - while let Some(gap) = gap_receiver.recv().await { - if handlers.len() >= gap_worker_count { - handlers.next().await; - } - - let client = gap_solana_rpc.clone(); - let sender = gap_sig_sender.clone(); - - let handle = spawn_crawl_worker(client, sender, gap); - - handlers.push(handle); - } - - futures::future::join_all(handlers).await; - }); - - let started = Instant::now(); - - let trees = if let Some(only_trees) = config.only_trees { - TreeResponse::find(&solana_rpc, only_trees).await? - } else { - TreeResponse::all(&solana_rpc).await? - }; - - let tree_count = trees.len(); - - info!( - "fetched {} trees in {}", - tree_count, - HumanDuration(started.elapsed()) - ); - - let tree_crawler_count = config.tree_crawler_count; - let mut crawl_handles = FuturesUnordered::new(); - - for tree in trees { - if crawl_handles.len() >= tree_crawler_count { - crawl_handles.next().await; - } - - let sender = gap_sender.clone(); - let pool = pool.clone(); - let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); - - let handle = spawn_gap_worker(conn, sender, tree); - - crawl_handles.push(handle); - } - - futures::future::try_join_all(crawl_handles).await?; - drop(gap_sender); - info!("crawled all trees"); - - gap_worker_manager.await?; - drop(sig_sender); - info!("all gaps processed"); - - transaction_worker_manager.await?; - info!("all transactions queued"); - - statsd_time!("job.completed", started.elapsed()); - - info!( - "crawled {} trees in {}", - tree_count, - HumanDuration(started.elapsed()) - ); - - Ok(()) -} - -fn spawn_gap_worker( - conn: DatabaseConnection, - sender: mpsc::Sender, - tree: TreeResponse, -) -> JoinHandle> { - tokio::spawn(async move { - let timing = Instant::now(); - - let mut gaps = TreeGapModel::find(&conn, tree.pubkey) - .await? - .into_iter() - .map(TryInto::try_into) - .collect::, _>>()?; - - let upper_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_desc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; - - let lower_known_seq = cl_audits_v2::Entity::find() - .filter(cl_audits_v2::Column::Tree.eq(tree.pubkey.as_ref().to_vec())) - .order_by_asc(cl_audits_v2::Column::Seq) - .one(&conn) - .await?; - - if let Some(upper_seq) = upper_known_seq { - let signature = Signature::try_from(upper_seq.tx.as_ref())?; - info!( - "tree {} has known highest seq {} filling tree from {}", - tree.pubkey, upper_seq.seq, signature - ); - gaps.push(TreeGapFill::new(tree.pubkey, None, Some(signature))); - } else if tree.seq > 0 { - info!( - "tree {} has no known highest seq but the actual seq is {} filling whole tree", - tree.pubkey, tree.seq - ); - gaps.push(TreeGapFill::new(tree.pubkey, None, None)); - } - - if let Some(lower_seq) = lower_known_seq.filter(|seq| seq.seq > 1) { - let signature = Signature::try_from(lower_seq.tx.as_ref())?; - - info!( - "tree {} has known lowest seq {} filling tree starting at {}", - tree.pubkey, lower_seq.seq, signature - ); - - gaps.push(TreeGapFill::new(tree.pubkey, Some(signature), None)); - } - - let gap_count = gaps.len(); - - for gap in gaps { - if let Err(e) = sender.send(gap).await { - statsd_count!("gap.failed", 1); - error!("send gap: {:?}", e); - } - } - - info!("crawling tree {} with {} gaps", tree.pubkey, gap_count); - - statsd_count!("tree.succeeded", 1); - statsd_time!("tree.crawled", timing.elapsed()); - - Ok::<(), anyhow::Error>(()) - }) -} - -fn spawn_crawl_worker( - client: Rpc, - sender: mpsc::Sender, - gap: TreeGapFill, -) -> JoinHandle<()> { - tokio::spawn(async move { - let timing = Instant::now(); - - if let Err(e) = gap.crawl(client, sender).await { - error!("tree transaction: {:?}", e); - - statsd_count!("gap.failed", 1); - } else { - statsd_count!("gap.succeeded", 1); - } - - statsd_time!("gap.queued", timing.elapsed()); - }) -} - -async fn queue_transaction<'a>( - client: Rpc, - queue: QueuePool, - signature: Signature, -) -> Result<(), TreeErrorKind> { - let transaction = client.get_transaction(&signature).await?; - - let message = seralize_encoded_transaction_with_status(FlatBufferBuilder::new(), transaction)?; - - queue - .push_transaction_backfill(message.finished_data()) - .await?; - - Ok(()) -} - -fn spawn_transaction_worker(client: Rpc, queue: QueuePool, signature: Signature) -> JoinHandle<()> { - tokio::spawn(async move { - let timing = Instant::now(); - - if let Err(e) = queue_transaction(client, queue, signature).await { - error!("queue transaction: {:?}", e); + let database_pool = connect_db(&config.database).await?; - statsd_count!("transaction.failed", 1); - } else { - statsd_count!("transaction.succeeded", 1); - } + let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumContext::new(database_pool, solana_rpc); - statsd_time!("transaction.queued", timing.elapsed()); - }) + start_backfill(context, config.backfill_bubblegum).await } diff --git a/ops/src/bubblegum/cmd.rs b/ops/src/bubblegum/cmd.rs index bb2244167..dd19c3246 100644 --- a/ops/src/bubblegum/cmd.rs +++ b/ops/src/bubblegum/cmd.rs @@ -1,4 +1,4 @@ -use super::{audit, backfiller}; +use super::{backfiller, replay, verify}; use anyhow::Result; use clap::{Args, Subcommand}; @@ -8,9 +8,10 @@ pub enum Commands { /// It crawls through trees and backfills any missed tree transactions. #[clap(name = "backfill")] Backfill(backfiller::Args), - /// The `audit` commands checks `cl_audits_v2` for any failed transactions and logs them to stdout. - #[clap(name = "audit")] - Audit(audit::Args), + #[clap(name = "replay")] + Replay(replay::Args), + /// The 'verify' command is used to verify the integrity of the bubblegum index. + Verify(verify::Args), } #[derive(Debug, Clone, Args)] @@ -24,8 +25,11 @@ pub async fn subcommand(subcommand: BubblegumCommand) -> Result<()> { Commands::Backfill(args) => { backfiller::run(args).await?; } - Commands::Audit(args) => { - audit::run(args).await?; + Commands::Replay(args) => { + replay::run(args).await?; + } + Commands::Verify(args) => { + verify::run(args).await?; } } diff --git a/ops/src/bubblegum/mod.rs b/ops/src/bubblegum/mod.rs index eb4c867ad..0f683a332 100644 --- a/ops/src/bubblegum/mod.rs +++ b/ops/src/bubblegum/mod.rs @@ -1,6 +1,6 @@ -mod audit; mod backfiller; mod cmd; -mod tree; +mod replay; +mod verify; pub use cmd::*; diff --git a/ops/src/bubblegum/replay.rs b/ops/src/bubblegum/replay.rs new file mode 100644 index 000000000..c777b913c --- /dev/null +++ b/ops/src/bubblegum/replay.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use clap::Parser; +use das_bubblegum::{start_bubblegum_replay, BubblegumContext, BubblegumReplayArgs}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, + + #[clap(flatten)] + pub replay_bubblegum: BubblegumReplayArgs, +} + +pub async fn run(config: Args) -> Result<()> { + let database_pool = connect_db(&config.database).await?; + + let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumContext::new(database_pool, solana_rpc); + + start_bubblegum_replay(context, config.replay_bubblegum).await +} diff --git a/ops/src/bubblegum/tree.rs b/ops/src/bubblegum/tree.rs deleted file mode 100644 index 09a3c92cb..000000000 --- a/ops/src/bubblegum/tree.rs +++ /dev/null @@ -1,278 +0,0 @@ -use anyhow::Result; -use borsh::BorshDeserialize; -use clap::Args; -use das_core::{QueuePoolError, Rpc}; -use log::error; -use sea_orm::{DatabaseConnection, DbBackend, FromQueryResult, Statement, Value}; -use solana_client::rpc_filter::{Memcmp, RpcFilterType}; -use solana_client::rpc_response::RpcConfirmedTransactionStatusWithSignature; -use solana_sdk::{account::Account, pubkey::Pubkey, signature::Signature}; -use spl_account_compression::id; -use spl_account_compression::state::{ - merkle_tree_get_size, ConcurrentMerkleTreeHeader, CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1, -}; -use std::str::FromStr; -use thiserror::Error as ThisError; -use tokio::sync::mpsc::Sender; - -const GET_SIGNATURES_FOR_ADDRESS_LIMIT: usize = 1000; - -#[derive(Debug, Clone, Args)] -pub struct ConfigBackfiller { - /// Solana RPC URL - #[arg(long, env)] - pub solana_rpc_url: String, -} - -#[derive(ThisError, Debug)] -pub enum TreeErrorKind { - #[error("solana rpc")] - Rpc(#[from] solana_client::client_error::ClientError), - #[error("anchor")] - Achor(#[from] anchor_client::anchor_lang::error::Error), - #[error("perkle serialize")] - PerkleSerialize(#[from] plerkle_serialization::error::PlerkleSerializationError), - #[error("perkle messenger")] - PlerkleMessenger(#[from] plerkle_messenger::MessengerError), - #[error("queue pool")] - QueuePool(#[from] QueuePoolError), - #[error("parse pubkey")] - ParsePubkey(#[from] solana_sdk::pubkey::ParsePubkeyError), - #[error("serialize tree response")] - SerializeTreeResponse, - #[error("sea orm")] - Database(#[from] sea_orm::DbErr), - #[error("try from pubkey")] - TryFromPubkey, - #[error("try from signature")] - TryFromSignature, -} - -const TREE_GAP_SQL: &str = r#" -WITH sequenced_data AS ( - SELECT - tree, - seq, - LEAD(seq) OVER (ORDER BY seq ASC) AS next_seq, - tx AS current_tx, - LEAD(tx) OVER (ORDER BY seq ASC) AS next_tx - FROM - cl_audits_v2 - WHERE - tree = $1 -), -gaps AS ( - SELECT - tree, - seq AS gap_start_seq, - next_seq AS gap_end_seq, - current_tx AS lower_bound_tx, - next_tx AS upper_bound_tx - FROM - sequenced_data - WHERE - next_seq IS NOT NULL AND - next_seq - seq > 1 -) -SELECT - tree, - gap_start_seq, - gap_end_seq, - lower_bound_tx, - upper_bound_tx -FROM - gaps -ORDER BY - gap_start_seq; -"#; - -#[derive(Debug, FromQueryResult, PartialEq, Clone)] -pub struct TreeGapModel { - pub tree: Vec, - pub gap_start_seq: i64, - pub gap_end_seq: i64, - pub lower_bound_tx: Vec, - pub upper_bound_tx: Vec, -} - -impl TreeGapModel { - pub async fn find(conn: &DatabaseConnection, tree: Pubkey) -> Result, TreeErrorKind> { - let statement = Statement::from_sql_and_values( - DbBackend::Postgres, - TREE_GAP_SQL, - vec![Value::Bytes(Some(Box::new(tree.as_ref().to_vec())))], - ); - - TreeGapModel::find_by_statement(statement) - .all(conn) - .await - .map_err(Into::into) - } -} - -impl TryFrom for TreeGapFill { - type Error = TreeErrorKind; - - fn try_from(model: TreeGapModel) -> Result { - let tree = Pubkey::try_from(model.tree).map_err(|_| TreeErrorKind::TryFromPubkey)?; - let upper = Signature::try_from(model.upper_bound_tx) - .map_err(|_| TreeErrorKind::TryFromSignature)?; - let lower = Signature::try_from(model.lower_bound_tx) - .map_err(|_| TreeErrorKind::TryFromSignature)?; - - Ok(Self::new(tree, Some(upper), Some(lower))) - } -} - -pub struct TreeGapFill { - tree: Pubkey, - before: Option, - until: Option, -} - -impl TreeGapFill { - pub fn new(tree: Pubkey, before: Option, until: Option) -> Self { - Self { - tree, - before, - until, - } - } - - pub async fn crawl(&self, client: Rpc, sender: Sender) -> Result<()> { - let mut before = self.before; - - loop { - let sigs = client - .get_signatures_for_address(&self.tree, before, self.until) - .await?; - let sig_count = sigs.len(); - - let successful_transactions = sigs - .into_iter() - .filter(|transaction| transaction.err.is_none()) - .collect::>(); - - for sig in successful_transactions.iter() { - let sig = Signature::from_str(&sig.signature)?; - - sender.send(sig).await?; - - before = Some(sig); - } - - if sig_count < GET_SIGNATURES_FOR_ADDRESS_LIMIT { - break; - } - } - - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub struct TreeHeaderResponse { - pub max_depth: u32, - pub max_buffer_size: u32, - pub creation_slot: u64, - pub size: usize, -} - -impl TryFrom for TreeHeaderResponse { - type Error = TreeErrorKind; - - fn try_from(payload: ConcurrentMerkleTreeHeader) -> Result { - let size = merkle_tree_get_size(&payload)?; - - Ok(Self { - max_depth: payload.get_max_depth(), - max_buffer_size: payload.get_max_buffer_size(), - creation_slot: payload.get_creation_slot(), - size, - }) - } -} - -#[derive(Debug, Clone)] -pub struct TreeResponse { - pub pubkey: Pubkey, - pub tree_header: TreeHeaderResponse, - pub seq: u64, -} - -impl TreeResponse { - pub fn try_from_rpc(pubkey: Pubkey, account: Account) -> Result { - let bytes = account.data.as_slice(); - - let (header_bytes, rest) = bytes.split_at(CONCURRENT_MERKLE_TREE_HEADER_SIZE_V1); - let header: ConcurrentMerkleTreeHeader = - ConcurrentMerkleTreeHeader::try_from_slice(header_bytes)?; - - let merkle_tree_size = merkle_tree_get_size(&header)?; - let (tree_bytes, _canopy_bytes) = rest.split_at(merkle_tree_size); - - let seq_bytes = tree_bytes[0..8].try_into()?; - let seq = u64::from_le_bytes(seq_bytes); - - let (auth, _) = Pubkey::find_program_address(&[pubkey.as_ref()], &mpl_bubblegum::ID); - - header.assert_valid_authority(&auth)?; - - let tree_header = header.try_into()?; - - Ok(Self { - pubkey, - tree_header, - seq, - }) - } - - pub async fn all(client: &Rpc) -> Result, TreeErrorKind> { - Ok(client - .get_program_accounts( - &id(), - Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes( - 0, - vec![1u8], - ))]), - ) - .await? - .into_iter() - .filter_map(|(pubkey, account)| Self::try_from_rpc(pubkey, account).ok()) - .collect()) - } - - pub async fn find(client: &Rpc, pubkeys: Vec) -> Result, TreeErrorKind> { - let pubkeys: Vec = pubkeys - .into_iter() - .map(|p| Pubkey::from_str(&p)) - .collect::, _>>()?; - let pubkey_batches = pubkeys.chunks(100); - let pubkey_batches_count = pubkey_batches.len(); - - let mut gma_handles = Vec::with_capacity(pubkey_batches_count); - - for batch in pubkey_batches { - gma_handles.push(async move { - let accounts = client.get_multiple_accounts(batch).await?; - - let results: Vec<(&Pubkey, Option)> = batch.iter().zip(accounts).collect(); - - Ok::<_, TreeErrorKind>(results) - }) - } - - let result = futures::future::try_join_all(gma_handles).await?; - - let trees = result - .into_iter() - .flatten() - .filter_map(|(pubkey, account)| { - account.map(|account| Self::try_from_rpc(*pubkey, account)) - }) - .collect::, _>>() - .map_err(|_| TreeErrorKind::SerializeTreeResponse)?; - - Ok(trees) - } -} diff --git a/ops/src/bubblegum/verify.rs b/ops/src/bubblegum/verify.rs new file mode 100644 index 000000000..46cb97cfc --- /dev/null +++ b/ops/src/bubblegum/verify.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use clap::Parser; +use das_bubblegum::{verify_bubblegum, BubblegumContext, VerifyArgs}; +use das_core::{connect_db, PoolArgs, Rpc, SolanaRpcArgs}; +use tracing::info; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + /// Verify Bubblegum Args + #[clap(flatten)] + pub verify_bubblegum: VerifyArgs, + + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, + + /// Solana configuration + #[clap(flatten)] + pub solana: SolanaRpcArgs, +} + +pub async fn run(config: Args) -> Result<()> { + let database_pool = connect_db(&config.database).await?; + + let solana_rpc = Rpc::from_config(&config.solana); + let context = BubblegumContext::new(database_pool, solana_rpc); + + let mut reports = verify_bubblegum(context, config.verify_bubblegum).await?; + + while let Some(report) = reports.recv().await { + info!( + "Tree: {}, Total Leaves: {}, Incorrect Proofs: {}, Not Found Proofs: {}, Correct Proofs: {}", + report.tree_pubkey, + report.total_leaves, + report.incorrect_proofs, + report.not_found_proofs, + report.correct_proofs + ); + } + + Ok(()) +} diff --git a/ops/src/main.rs b/ops/src/main.rs index 09e73be12..0b80c5d1b 100644 --- a/ops/src/main.rs +++ b/ops/src/main.rs @@ -1,5 +1,6 @@ mod account; mod bubblegum; +mod metadata; use account::{subcommand as account_subcommand, AccountCommand}; use anyhow::Result; @@ -19,6 +20,8 @@ enum Command { Bubblegum(BubblegumCommand), #[clap(name = "account")] Account(AccountCommand), + #[clap(name = "metadata_json")] + MetadataJson(metadata::MetadataJsonCommand), } #[tokio::main] @@ -30,6 +33,7 @@ async fn main() -> Result<()> { match args.command { Command::Bubblegum(subcommand) => bubblegum_subcommand(subcommand).await?, Command::Account(subcommand) => account_subcommand(subcommand).await?, + Command::MetadataJson(subcommand) => metadata::subcommand(subcommand).await?, } Ok(()) diff --git a/ops/src/metadata/backfiller.rs b/ops/src/metadata/backfiller.rs new file mode 100644 index 000000000..9557e950e --- /dev/null +++ b/ops/src/metadata/backfiller.rs @@ -0,0 +1,160 @@ +use anyhow::{Ok, Result}; +use clap::Parser; +use das_core::{ + connect_db, perform_metadata_json_task, DownloadMetadataInfo, DownloadMetadataJsonRetryConfig, + MetadataJsonDownloadWorkerArgs, PoolArgs, +}; + +use digital_asset_types::dao::asset_data; +use indicatif::HumanDuration; +use log::{debug, error}; +use reqwest::Client; +use sea_orm::{ColumnTrait, EntityTrait, JsonValue, QueryFilter, SqlxPostgresConnector}; +use std::sync::Arc; +use tokio::{task::JoinHandle, time::Instant}; + +#[derive(Debug, Parser, Clone)] +pub struct Args { + #[clap(flatten)] + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, + /// Database configuration + #[clap(flatten)] + pub database: PoolArgs, +} + +pub const DEFAULT_METADATA_JSON_DOWNLOAD_WORKER_COUNT: usize = 25; + +#[derive(Debug, Clone)] +pub struct MetadataJsonBackfillerContext { + pub database_pool: sqlx::PgPool, + pub metadata_json_download_worker: MetadataJsonDownloadWorkerArgs, +} + +pub async fn start_backfill(context: MetadataJsonBackfillerContext) -> Result<()> { + let MetadataJsonBackfillerContext { + database_pool, + metadata_json_download_worker: + MetadataJsonDownloadWorkerArgs { + metadata_json_download_worker_count, + metadata_json_download_worker_request_timeout, + }, + } = context; + + let mut worker_count = if metadata_json_download_worker_count > 0 { + metadata_json_download_worker_count + } else { + DEFAULT_METADATA_JSON_DOWNLOAD_WORKER_COUNT + }; + let database_pool = database_pool.clone(); + let conn = SqlxPostgresConnector::from_sqlx_postgres_pool(database_pool.clone()); + + let download_metadata_info_vec = asset_data::Entity::find() + .filter(asset_data::Column::Metadata.eq(JsonValue::String("processing".to_string()))) + .all(&conn) + .await? + .iter() + .map(|d| DownloadMetadataInfo::new(d.id.clone(), d.metadata_url.clone(), d.slot_updated)) + .collect::>(); + + let metadata_vec_len = download_metadata_info_vec.len(); + debug!( + "Found {} assets to download", + download_metadata_info_vec.len() + ); + + if metadata_vec_len == 0 { + return Ok(()); + } + + if worker_count > metadata_vec_len { + if metadata_vec_len == 1 { + worker_count = 1; + } else { + // If the number of assets is less than the number of workers, we assume each worker will handle 2 assets + worker_count = metadata_vec_len / 2; + } + } + + let excess_tasks = metadata_vec_len % worker_count; + let mut current_tasks_per_worker = if excess_tasks > 0 { + metadata_vec_len / worker_count + 1 + } else { + metadata_vec_len / worker_count + }; + + let mut handlers: Vec> = Vec::with_capacity(metadata_json_download_worker_count); + + let mut curr_start = 0; + let client = Client::builder() + .timeout(std::time::Duration::from_millis( + metadata_json_download_worker_request_timeout, + )) + .build()?; + + debug!("worker_count: {}", worker_count); + for _ in 0..worker_count { + let start = curr_start; + + let end = start + current_tasks_per_worker; + + let handler = spawn_metadata_fetch_task( + client.clone(), + database_pool.clone(), + &download_metadata_info_vec[start..end], + ); + + handlers.push(handler); + + current_tasks_per_worker = current_tasks_per_worker.saturating_sub(1); + + curr_start = end; + } + + futures::future::join_all(handlers).await; + + Ok(()) +} + +fn spawn_metadata_fetch_task( + client: reqwest::Client, + pool: sqlx::PgPool, + download_metadata_info: &[DownloadMetadataInfo], +) -> JoinHandle<()> { + let download_metadata_info = download_metadata_info.to_vec(); + tokio::spawn(async move { + for d in download_metadata_info.iter() { + let timing = Instant::now(); + let asset_data_id = bs58::encode(d.asset_data_id.clone()).into_string(); + + if let Err(e) = perform_metadata_json_task( + client.clone(), + pool.clone(), + d, + Arc::new(DownloadMetadataJsonRetryConfig::default()), + ) + .await + { + error!("Asset {} failed: {}", asset_data_id, e); + } + + debug!( + "Asset {} finished in {}", + asset_data_id, + HumanDuration(timing.elapsed()) + ); + } + }) +} + +pub async fn run(config: Args) -> Result<()> { + let database_pool = connect_db(&config.database).await?; + + let context = MetadataJsonBackfillerContext { + database_pool, + metadata_json_download_worker: config.metadata_json_download_worker, + }; + + start_backfill(context).await?; + + Ok(()) +} diff --git a/ops/src/metadata/cmd.rs b/ops/src/metadata/cmd.rs new file mode 100644 index 000000000..e2b26be8b --- /dev/null +++ b/ops/src/metadata/cmd.rs @@ -0,0 +1,27 @@ +use super::backfiller; +use anyhow::Result; +use clap::{Args, Subcommand}; + +#[derive(Debug, Clone, Subcommand)] +pub enum Commands { + /// The 'backfill' command is used to cross-reference the index against on-chain accounts. + /// It fetches all metadata json data marked as 'processing' and downloads the metadata json files. + #[clap(name = "backfill")] + Backfill(backfiller::Args), +} + +#[derive(Debug, Clone, Args)] +pub struct MetadataJsonCommand { + #[clap(subcommand)] + pub action: Commands, +} + +pub async fn subcommand(subcommand: MetadataJsonCommand) -> Result<()> { + match subcommand.action { + Commands::Backfill(args) => { + backfiller::run(args).await?; + } + } + + Ok(()) +} diff --git a/ops/src/metadata/mod.rs b/ops/src/metadata/mod.rs new file mode 100644 index 000000000..d798b6e9e --- /dev/null +++ b/ops/src/metadata/mod.rs @@ -0,0 +1,4 @@ +mod backfiller; +mod cmd; + +pub use cmd::*; diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index de6eb6cc3..4262824a5 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -10,12 +10,17 @@ blockbuster = { workspace = true } bs58 = { workspace = true } cadence = { workspace = true } cadence-macros = { workspace = true } -digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] } +das-core = { workspace = true } +digital_asset_types = { workspace = true, features = [ + "json_types", + "sql_types", +] } futures = { workspace = true } heck = { workspace = true } mpl-bubblegum = { workspace = true } num-traits = { workspace = true } sea-orm = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index b086c6de8..470fab631 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -1,8 +1,8 @@ use { crate::error::{ProgramTransformerError, ProgramTransformerResult}, + das_core::DownloadMetadataInfo, digital_asset_types::dao::{ - asset, asset_authority, asset_creators, asset_data, asset_grouping, backfill_items, - cl_audits_v2, cl_items, + asset, asset_authority, asset_creators, asset_data, asset_grouping, cl_audits_v2, cl_items, sea_orm_active_enums::{ ChainMutability, Instruction, Mutability, OwnerType, RoyaltyTargetType, SpecificationAssetClass, SpecificationVersions, @@ -10,14 +10,14 @@ use { }, mpl_bubblegum::types::{Collection, Creator}, sea_orm::{ - entity::{ActiveValue, ColumnTrait, EntityTrait}, + entity::{ActiveValue, EntityTrait}, prelude::*, - query::{JsonValue, QueryFilter, QuerySelect, QueryTrait}, + query::{JsonValue, QueryTrait}, sea_query::query::OnConflict, ConnectionTrait, DbBackend, TransactionTrait, }, spl_account_compression::events::ChangeLogEventV1, - tracing::{debug, error, info}, + tracing::{debug, error}, }; pub async fn save_changelog_event<'c, T>( @@ -40,7 +40,7 @@ const fn node_idx_to_leaf_idx(index: i64, tree_height: u32) -> i64 { pub async fn insert_change_log<'c, T>( change_log_event: &ChangeLogEventV1, - slot: u64, + _slot: u64, txn_id: &str, txn: &T, instruction: &str, @@ -48,10 +48,9 @@ pub async fn insert_change_log<'c, T>( where T: ConnectionTrait + TransactionTrait, { - let mut i: i64 = 0; let depth = change_log_event.path.len() - 1; let tree_id = change_log_event.id.as_ref(); - for p in change_log_event.path.iter() { + for (i, p) in (0_i64..).zip(change_log_event.path.iter()) { let node_idx = p.index as i64; debug!( "seq {}, index {} level {}, node {:?}, txn: {:?}, instruction {}", @@ -78,7 +77,6 @@ where ..Default::default() }; - i += 1; let mut query = cl_items::Entity::insert(item) .on_conflict( OnConflict::columns([cl_items::Column::Tree, cl_items::Column::NodeIdx]) @@ -131,37 +129,6 @@ where } } - // If and only if the entire path of nodes was inserted into the `cl_items` table, then insert - // a single row into the `backfill_items` table. This way if an incomplete path was inserted - // into `cl_items` due to an error, a gap will be created for the tree and the backfiller will - // fix it. - if i - 1 == depth as i64 { - // See if the tree already exists in the `backfill_items` table. - let rows = backfill_items::Entity::find() - .filter(backfill_items::Column::Tree.eq(tree_id)) - .limit(1) - .all(txn) - .await?; - - // If the tree does not exist in `backfill_items` and the sequence number is greater than 1, - // then we know we will need to backfill the tree from sequence number 1 up to the current - // sequence number. So in this case we set at flag to force checking the tree. - let force_chk = rows.is_empty() && change_log_event.seq > 1; - - info!("Adding to backfill_items table at level {}", i - 1); - let item = backfill_items::ActiveModel { - tree: ActiveValue::Set(tree_id.to_vec()), - seq: ActiveValue::Set(change_log_event.seq as i64), - slot: ActiveValue::Set(slot as i64), - force_chk: ActiveValue::Set(force_chk), - backfilled: ActiveValue::Set(false), - failed: ActiveValue::Set(false), - ..Default::default() - }; - - backfill_items::Entity::insert(item).exec(txn).await?; - } - Ok(()) } @@ -403,13 +370,11 @@ pub async fn upsert_asset_data( chain_data: JsonValue, metadata_url: String, metadata_mutability: Mutability, - metadata: JsonValue, slot_updated: i64, - reindex: Option, raw_name: Vec, raw_symbol: Vec, seq: i64, -) -> ProgramTransformerResult<()> +) -> ProgramTransformerResult> where T: ConnectionTrait + TransactionTrait, { @@ -417,11 +382,11 @@ where id: ActiveValue::Set(id.clone()), chain_data_mutability: ActiveValue::Set(chain_data_mutability), chain_data: ActiveValue::Set(chain_data), - metadata_url: ActiveValue::Set(metadata_url), + metadata_url: ActiveValue::Set(metadata_url.clone()), metadata_mutability: ActiveValue::Set(metadata_mutability), - metadata: ActiveValue::Set(metadata), + metadata: ActiveValue::Set(JsonValue::String("processing".to_string())), slot_updated: ActiveValue::Set(slot_updated), - reindex: ActiveValue::Set(reindex), + reindex: ActiveValue::Set(Some(true)), raw_name: ActiveValue::Set(Some(raw_name)), raw_symbol: ActiveValue::Set(Some(raw_symbol)), base_info_seq: ActiveValue::Set(Some(seq)), @@ -435,9 +400,7 @@ where asset_data::Column::ChainData, asset_data::Column::MetadataUrl, asset_data::Column::MetadataMutability, - // Don't update asset_data::Column::Metadata if it already exists. Even if we - // are indexing `update_metadata`` and there's a new URI, the new background - // task will overwrite it. + asset_data::Column::Metadata, asset_data::Column::SlotUpdated, asset_data::Column::Reindex, asset_data::Column::RawName, @@ -450,15 +413,27 @@ where // Do not overwrite changes that happened after decompression (asset_data.base_info_seq = 0). // Do not overwrite changes from a later Bubblegum instruction. + // Do not update the record if the incoming slot is larger than the current or if it's null. + // Update if the current slot on the record is null. query.sql = format!( - "{} WHERE (asset_data.base_info_seq != 0 AND excluded.base_info_seq >= asset_data.base_info_seq) OR asset_data.base_info_seq IS NULL", + "{} WHERE ((asset_data.base_info_seq != 0 AND excluded.base_info_seq >= asset_data.base_info_seq) OR asset_data.base_info_seq IS NULL) AND (excluded.slot_updated <= asset_data.slot_updated OR asset_data.slot_updated IS NULL)", query.sql ); - txn.execute(query) + + let result = txn + .execute(query) .await .map_err(|db_err| ProgramTransformerError::StorageWriteError(db_err.to_string()))?; - Ok(()) + if result.rows_affected() > 0 { + Ok(Some(DownloadMetadataInfo::new( + id, + metadata_url, + slot_updated, + ))) + } else { + Ok(None) + } } #[allow(clippy::too_many_arguments)] @@ -529,7 +504,7 @@ where pub async fn upsert_asset_creators( txn: &T, id: Vec, - creators: &Vec, + creators: &[Creator], slot_updated: i64, seq: i64, ) -> ProgramTransformerResult<()> diff --git a/program_transformers/src/bubblegum/mint_v1.rs b/program_transformers/src/bubblegum/mint_v1.rs index fb179727a..47193ee1f 100644 --- a/program_transformers/src/bubblegum/mint_v1.rs +++ b/program_transformers/src/bubblegum/mint_v1.rs @@ -24,7 +24,7 @@ use { }, json::ChainDataV1, }, - sea_orm::{query::JsonValue, ConnectionTrait, TransactionTrait}, + sea_orm::{ConnectionTrait, TransactionTrait}, tracing::warn, }; @@ -91,16 +91,14 @@ where // automatically rolled back. let multi_txn = txn.begin().await?; - upsert_asset_data( + let download_metadata_info = upsert_asset_data( &multi_txn, id_bytes.to_vec(), chain_mutability, chain_data_json, uri.clone(), Mutability::Mutable, - JsonValue::String("processing".to_string()), slot_i, - Some(true), name.to_vec(), symbol.to_vec(), seq as i64, @@ -207,7 +205,7 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(id_bytes.to_vec(), uri))) + Ok(download_metadata_info) } _ => Err(ProgramTransformerError::NotImplemented), }; diff --git a/program_transformers/src/bubblegum/mod.rs b/program_transformers/src/bubblegum/mod.rs index 5cdff2b8a..03a800858 100644 --- a/program_transformers/src/bubblegum/mod.rs +++ b/program_transformers/src/bubblegum/mod.rs @@ -11,7 +11,7 @@ use { token_metadata::types::UseMethod as TokenMetadataUseMethod, }, sea_orm::{ConnectionTrait, TransactionTrait}, - tracing::{debug, info}, + tracing::debug, }; mod burn; @@ -58,7 +58,7 @@ where InstructionName::SetDecompressibleState => "SetDecompressibleState", InstructionName::UpdateMetadata => "UpdateMetadata", }; - info!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id); + debug!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id); match ix_type { InstructionName::Transfer => { diff --git a/program_transformers/src/bubblegum/transfer.rs b/program_transformers/src/bubblegum/transfer.rs index 9c551beea..617efdf72 100644 --- a/program_transformers/src/bubblegum/transfer.rs +++ b/program_transformers/src/bubblegum/transfer.rs @@ -77,6 +77,7 @@ where } } } + Err(ProgramTransformerError::ParsingError( "Ix not parsed correctly".to_string(), )) diff --git a/program_transformers/src/bubblegum/update_metadata.rs b/program_transformers/src/bubblegum/update_metadata.rs index 702aa552c..dff719fd4 100644 --- a/program_transformers/src/bubblegum/update_metadata.rs +++ b/program_transformers/src/bubblegum/update_metadata.rs @@ -22,7 +22,7 @@ use { }, json::ChainDataV1, }, - sea_orm::{query::*, ConnectionTrait, JsonValue}, + sea_orm::{query::*, ConnectionTrait}, tracing::warn, }; @@ -114,16 +114,14 @@ where // automatically rolled back. let multi_txn = txn.begin().await?; - upsert_asset_data( + let download_metadata_info = upsert_asset_data( &multi_txn, id_bytes.to_vec(), chain_mutability, chain_data_json, uri.clone(), Mutability::Mutable, - JsonValue::String("processing".to_string()), slot_i, - Some(true), name.into_bytes().to_vec(), symbol.into_bytes().to_vec(), seq as i64, @@ -188,7 +186,7 @@ where return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(id_bytes.to_vec(), uri))) + Ok(download_metadata_info) } _ => Err(ProgramTransformerError::NotImplemented), }; diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index ee20aa2b4..c516b9cc9 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -17,11 +17,12 @@ use { ProgramParseResult, }, }, - futures::future::BoxFuture, + das_core::{DownloadMetadataInfo, DownloadMetadataNotifier}, sea_orm::{ entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, SqlxPostgresConnector, TransactionTrait, }, + serde::Deserialize, serde_json::{Map, Value}, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, @@ -33,7 +34,7 @@ use { }; mod asset_upserts; -mod bubblegum; +pub mod bubblegum; pub mod error; mod mpl_core_program; mod token; @@ -41,7 +42,7 @@ mod token_extensions; mod token_inscription; mod token_metadata; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct AccountInfo { pub slot: u64, pub pubkey: Pubkey, @@ -49,7 +50,7 @@ pub struct AccountInfo { pub data: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct TransactionInfo { pub slot: u64, pub signature: Signature, @@ -58,33 +59,6 @@ pub struct TransactionInfo { pub meta_inner_instructions: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DownloadMetadataInfo { - asset_data_id: Vec, - uri: String, -} - -impl DownloadMetadataInfo { - pub fn new(asset_data_id: Vec, uri: String) -> Self { - Self { - asset_data_id, - uri: uri.trim().replace('\0', ""), - } - } - - pub fn into_inner(self) -> (Vec, String) { - (self.asset_data_id, self.uri) - } -} - -pub type DownloadMetadataNotifier = Box< - dyn Fn( - DownloadMetadataInfo, - ) -> BoxFuture<'static, Result<(), Box>> - + Sync - + Send, ->; - pub struct ProgramTransformer { storage: DatabaseConnection, download_metadata_notifier: DownloadMetadataNotifier, diff --git a/program_transformers/src/mpl_core_program/v1_asset.rs b/program_transformers/src/mpl_core_program/v1_asset.rs index a022bc67e..e4779d92f 100644 --- a/program_transformers/src/mpl_core_program/v1_asset.rs +++ b/program_transformers/src/mpl_core_program/v1_asset.rs @@ -28,7 +28,7 @@ use { query::{JsonValue, QueryFilter, QueryTrait}, sea_query::query::OnConflict, sea_query::Expr, - ConnectionTrait, CursorTrait, DbBackend, TransactionTrait, + ConnectionTrait, CursorTrait, DbBackend, Statement, TransactionTrait, }, serde_json::{value::Value, Map}, solana_sdk::pubkey::Pubkey, @@ -108,6 +108,16 @@ pub async fn save_v1_asset( let txn = conn.begin().await?; + let set_lock_timeout = "SET LOCAL lock_timeout = '5s';"; + let set_local_app_name = + "SET LOCAL application_name = 'das::program_transformers::mpl_core_program::v1_asset';"; + let set_lock_timeout_stmt = + Statement::from_string(txn.get_database_backend(), set_lock_timeout.to_string()); + let set_local_app_name_stmt = + Statement::from_string(txn.get_database_backend(), set_local_app_name.to_string()); + txn.execute(set_lock_timeout_stmt).await?; + txn.execute(set_local_app_name_stmt).await?; + let model = asset_authority::ActiveModel { asset_id: ActiveValue::Set(id_vec.clone()), authority: ActiveValue::Set(update_authority.clone()), @@ -461,7 +471,7 @@ pub async fn save_v1_asset( } // Otherwise return with info for background downloading. - Ok(Some(DownloadMetadataInfo::new(id_vec.clone(), uri))) + Ok(Some(DownloadMetadataInfo::new(id_vec.clone(), uri, slot_i))) } // Modify the JSON structure to remove the `Plugin` name and just display its data. diff --git a/program_transformers/src/token_extensions/mod.rs b/program_transformers/src/token_extensions/mod.rs index b68783011..7b18a0eeb 100644 --- a/program_transformers/src/token_extensions/mod.rs +++ b/program_transformers/src/token_extensions/mod.rs @@ -273,10 +273,11 @@ async fn upsert_asset_data( return Ok(None); } - Ok(Some(DownloadMetadataInfo { - uri: metadata.uri.clone(), - asset_data_id: key_bytes, - })) + Ok(Some(DownloadMetadataInfo::new( + key_bytes, + metadata.uri.clone(), + slot, + ))) } struct AssetMetadataAccountCols { diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index 70634f5bc..b3daf49b2 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -30,7 +30,7 @@ use { entity::{ActiveValue, ColumnTrait, EntityTrait}, query::{JsonValue, Order, QueryFilter, QueryOrder, QueryTrait}, sea_query::query::OnConflict, - ConnectionTrait, DbBackend, DbErr, TransactionTrait, + ConnectionTrait, DbBackend, DbErr, Statement, TransactionTrait, }, solana_sdk::pubkey, solana_sdk::pubkey::Pubkey, @@ -97,11 +97,11 @@ pub async fn index_and_fetch_mint_data( .map_err(|db_err| ProgramTransformerError::AssetIndexError(db_err.to_string()))?; Ok(Some(token)) } else { - warn!( - target: "Mint not found", - "Mint not found in 'tokens' table for mint {}", - bs58::encode(&mint_pubkey_vec).into_string() - ); + // warn!( + // target: "Mint not found", + // "Mint not found in 'tokens' table for mint {}", + // bs58::encode(&mint_pubkey_vec).into_string() + // ); Ok(None) } } @@ -244,6 +244,17 @@ pub async fn save_v1_asset( base_info_seq: ActiveValue::Set(Some(0)), }; let txn = conn.begin().await?; + + let set_lock_timeout = "SET LOCAL lock_timeout = '5s';"; + let set_local_app_name = + "SET LOCAL application_name = 'das::program_transformers::token_metadata::v1_asset';"; + let set_lock_timeout_stmt = + Statement::from_string(txn.get_database_backend(), set_lock_timeout.to_string()); + let set_local_app_name_stmt = + Statement::from_string(txn.get_database_backend(), set_local_app_name.to_string()); + txn.execute(set_lock_timeout_stmt).await?; + txn.execute(set_local_app_name_stmt).await?; + let mut query = asset_data::Entity::insert(asset_data_model) .on_conflict( OnConflict::columns([asset_data::Column::Id]) @@ -423,7 +434,11 @@ pub async fn save_v1_asset( return Ok(None); } - Ok(Some(DownloadMetadataInfo::new(mint_pubkey_vec, uri))) + Ok(Some(DownloadMetadataInfo::new( + mint_pubkey_vec, + uri, + slot_i, + ))) } async fn upsert_asset_v1_account_attachments( diff --git a/prometheus-config.yaml b/prometheus-config.yaml new file mode 100644 index 000000000..3f975ab85 --- /dev/null +++ b/prometheus-config.yaml @@ -0,0 +1,14 @@ +global: + scrape_interval: 1s + evaluation_interval: 5s + +scrape_configs: + - job_name: "prometheus" + honor_labels: true + static_configs: + - targets: + [ + "host.docker.internal:8873", + "host.docker.internal:8875", + "host.docker.internal:8876", + ] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eb84afdfc..402b06845 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.75.0" +channel = "1.79.0" components = ["clippy", "rustfmt"] targets = [] profile = "minimal" diff --git a/tools/acc_forwarder/src/main.rs b/tools/acc_forwarder/src/main.rs index 6e83e6e21..20a8b97b1 100644 --- a/tools/acc_forwarder/src/main.rs +++ b/tools/acc_forwarder/src/main.rs @@ -1,3 +1,4 @@ +#![allow(deprecated)] use { anyhow::Context, clap::Parser,