diff --git a/Builder.Dockerfile b/Builder.Dockerfile index 4ea85b7c9..e93234724 100644 --- a/Builder.Dockerfile +++ b/Builder.Dockerfile @@ -5,6 +5,7 @@ RUN apt-get update -y && \ RUN mkdir /rust RUN mkdir /rust/bins COPY Cargo.toml /rust +COPY Cargo.lock /rust COPY core /rust/core COPY das_api /rust/das_api COPY digital_asset_types /rust/digital_asset_types diff --git a/Cargo.lock b/Cargo.lock index 5563779d4..c58b6fee4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "getrandom 0.2.10", @@ -396,6 +396,20 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "aquamarine" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da02abba9f9063d786eab1509833ebb2fac0f966862ca59439c76b9c566760" +dependencies = [ + "include_dir", + "itertools 0.10.5", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -605,13 +619,12 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 3.1.0", - "event-listener-strategy", + "event-listener-strategy 0.5.2", "futures-core", "pin-project-lite", ] @@ -693,7 +706,7 @@ dependencies = [ "futures-lite 2.0.1", "parking", "polling 3.4.0", - "rustix 0.38.18", + "rustix 0.38.34", "slab", "tracing", "windows-sys 0.52.0", @@ -715,7 +728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" dependencies = [ "event-listener 3.1.0", - "event-listener-strategy", + "event-listener-strategy 0.3.0", "pin-project-lite", ] @@ -785,9 +798,9 @@ checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -889,9 +902,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -951,9 +964,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" dependencies = [ "serde", ] @@ -1043,7 +1056,7 @@ dependencies = [ "solana-transaction-status", "solana-zk-token-sdk", "spl-account-compression", - "spl-concurrent-merkle-tree", + "spl-concurrent-merkle-tree 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "spl-noop", "spl-pod", "spl-token", @@ -1059,7 +1072,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.1.0", + "async-channel 2.3.1", "async-lock 3.1.1", "async-task", "fastrand 2.0.1", @@ -1089,6 +1102,16 @@ dependencies = [ "hashbrown 0.13.2", ] +[[package]] +name = "borsh" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +dependencies = [ + "borsh-derive 1.5.1", + "cfg_aliases", +] + [[package]] name = "borsh-derive" version = "0.9.3" @@ -1115,6 +1138,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +dependencies = [ + "once_cell", + "proc-macro-crate 3.1.0", + "proc-macro2", + "quote", + "syn 2.0.66", + "syn_derive", +] + [[package]] name = "borsh-derive-internal" version = "0.9.3" @@ -1245,9 +1282,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" dependencies = [ "bytemuck_derive", ] @@ -1275,6 +1312,27 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cadence" version = "0.29.1" @@ -1319,6 +1377,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.31" @@ -1334,6 +1398,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "chrono-humanize" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799627e6b4d27827a814e837b9d8a504832086081806d45b1afa34dc982b023b" +dependencies = [ + "chrono", +] + [[package]] name = "cipher" version = "0.3.0" @@ -1467,7 +1540,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util", + "tokio-util 0.7.9", ] [[package]] @@ -1481,15 +1554,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -1575,11 +1648,10 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ - "cfg-if", "crossbeam-utils", ] @@ -1619,12 +1691,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -1798,16 +1867,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if", - "num_cpus", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -1819,6 +1878,7 @@ dependencies = [ "lock_api", "once_cell", "parking_lot_core 0.9.8", + "rayon", ] [[package]] @@ -1894,6 +1954,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.9.0" @@ -1937,12 +2003,21 @@ dependencies = [ "serde", "serde_json", "solana-sdk", - "spl-concurrent-merkle-tree", + "spl-concurrent-merkle-tree 0.2.0", "thiserror", "tokio", "url", ] +[[package]] +name = "dir-diff" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ad16bf5f84253b50d6557681c58c3ab67c47c77d39fed9aeb56e947290bd10" +dependencies = [ + "walkdir", +] + [[package]] name = "dirs" version = "4.0.0" @@ -2003,6 +2078,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dyn-clone" version = "1.0.14" @@ -2050,6 +2131,18 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "educe" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "either" version = "1.9.0" @@ -2076,19 +2169,32 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "enum-ordinalize" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" dependencies = [ + "num-bigint 0.4.4", + "num-traits", "proc-macro2", "quote", "syn 2.0.66", @@ -2128,12 +2234,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2176,6 +2282,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener 5.3.0", + "pin-project-lite", +] + [[package]] name = "fake" version = "2.9.2" @@ -2238,6 +2354,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + [[package]] name = "finl_unicode" version = "1.2.0" @@ -2256,14 +2384,23 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2287,13 +2424,19 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -2323,9 +2466,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2338,9 +2481,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2348,15 +2491,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2376,9 +2519,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2411,9 +2554,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -2422,21 +2565,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2560,7 +2703,7 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.9", "tracing", ] @@ -2597,7 +2740,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", ] [[package]] @@ -2606,7 +2749,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", "allocator-api2", ] @@ -2788,7 +2931,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", ] @@ -2837,9 +2980,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2861,6 +3004,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "include_dir" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18762faeff7122e89e0857b02f7ce6fcc0d101d5e9ad2ad7846cc01d61b7f19e" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "index_list" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70891286cb8e844fdfcf1178b47569699f9e20b5ecc4b45a6240a64771444638" + [[package]] name = "indexmap" version = "1.9.3" @@ -2874,9 +3042,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.1", @@ -2930,6 +3098,7 @@ name = "integration_tests" version = "0.1.0" dependencies = [ "anyhow", + "async-channel 2.3.1", "borsh 0.10.3", "das_api", "digital_asset_types", @@ -2940,6 +3109,7 @@ dependencies = [ "itertools 0.10.5", "log", "migration", + "mpl-bubblegum", "mpl-token-metadata", "nft_ingester", "once_cell", @@ -2953,8 +3123,10 @@ dependencies = [ "solana-client", "solana-sdk", "solana-transaction-status", + "spl-concurrent-merkle-tree 0.2.0", "spl-token", "sqlx", + "tempfile", "tokio", "tokio-stream", ] @@ -2983,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", - "rustix 0.38.18", + "rustix 0.38.34", "windows-sys 0.48.0", ] @@ -3022,9 +3194,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -3124,7 +3296,7 @@ dependencies = [ "soketto", "tokio", "tokio-stream", - "tokio-util", + "tokio-util 0.7.9", "tower", "tracing", ] @@ -3150,6 +3322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a26f49495f94a283312e7ef45a243540ef20c9356bb01c8d84a61ac8ba5339b" dependencies = [ "borsh 0.10.3", + "serde", ] [[package]] @@ -3178,9 +3351,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libm" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libsecp256k1" @@ -3256,9 +3435,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "load_generation" @@ -3294,6 +3473,35 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -3435,15 +3643,42 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "modular-bitfield" version = "0.11.2" @@ -3467,23 +3702,32 @@ dependencies = [ [[package]] name = "mpl-bubblegum" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3cbca5deb859e66a1a21ada94f2eaab3eb5caa4584c0c8ade0efac29a5414b8" +version = "1.4.0" dependencies = [ + "assert_matches", "borsh 0.10.3", + "bytemuck", "kaigan", "num-derive 0.3.3", "num-traits", + "serde", + "serde_with 3.8.1", "solana-program", + "solana-program-test", + "solana-sdk", + "spl-account-compression", + "spl-associated-token-account", + "spl-concurrent-merkle-tree 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spl-merkle-tree-reference 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spl-noop", + "spl-token", "thiserror", ] [[package]] name = "mpl-core" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b314fcf76dc22cc42a02a208231c667d0c533ce140d6ba9985df5a2a50b0aa" +source = "git+https://github.com/RequescoS/mpl-core.git#00496a76af1745aa2349199367be245cba2fec59" dependencies = [ "base64 0.22.0", "borsh 0.10.3", @@ -3493,7 +3737,7 @@ dependencies = [ "rmp-serde", "serde", "serde_json", - "serde_with 3.6.1", + "serde_with 3.8.1", "solana-program", "thiserror", ] @@ -3508,7 +3752,7 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "serde", - "serde_with 3.6.1", + "serde_with 3.8.1", "solana-program", "thiserror", ] @@ -3535,6 +3779,7 @@ dependencies = [ name = "nft_ingester" version = "0.7.2" dependencies = [ + "async-channel 2.3.1", "async-trait", "blockbuster", "borsh 0.10.3", @@ -3568,6 +3813,7 @@ dependencies = [ "thiserror", "tokio", "tracing-subscriber", + "triomphe", "url", ] @@ -3594,6 +3840,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3712,6 +3964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3760,7 +4013,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 2.0.66", @@ -3832,7 +4085,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -3870,6 +4123,25 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "thiserror", +] + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -4017,9 +4289,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "percentage" @@ -4118,8 +4390,7 @@ dependencies = [ [[package]] name = "plerkle_serialization" version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f832646491065468aa8e222b47d41dd5250e4be7866725bef5f0d31c64538f5f" +source = "git+https://github.com/n00m4d/digital-asset-validator-plugin.git#6920f6d3f30d6bc9c9c7eefeb5c7c1bcc87fa0c8" dependencies = [ "bs58 0.4.0", "chrono", @@ -4155,7 +4426,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.18", + "rustix 0.38.34", "tracing", "windows-sys 0.52.0", ] @@ -4203,12 +4474,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] -name = "proc-macro-crate" -version = "0.1.5" +name = "predicates" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ - "toml 0.5.11", + "difflib", + "float-cmp", + "itertools 0.10.5", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml 0.5.11", ] [[package]] @@ -4221,6 +4522,15 @@ dependencies = [ "toml_edit 0.19.15", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.0", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4271,6 +4581,10 @@ dependencies = [ name = "program_transformers" version = "0.7.2" dependencies = [ + "anchor-lang", + "async-channel 2.3.1", + "async-trait", + "bincode", "blockbuster", "bs58 0.4.0", "cadence", @@ -4278,18 +4592,29 @@ dependencies = [ "digital_asset_types", "futures", "heck 0.5.0", + "hex", + "mockall", "mpl-bubblegum", "num-traits", + "paste", + "rand 0.8.5", + "reqwest", "sea-orm", + "serde", + "serde_derive", "serde_json", + "serde_with 3.8.1", "solana-sdk", "solana-transaction-status", "spl-account-compression", + "spl-concurrent-merkle-tree 0.2.0", "spl-token", "sqlx", "thiserror", "tokio", "tracing", + "triomphe", + "xxhash-rust", ] [[package]] @@ -4374,7 +4699,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.12", "thiserror", "tokio", "tracing", @@ -4390,7 +4715,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.7", + "rustls 0.21.12", "rustls-native-certs", "slab", "thiserror", @@ -4406,7 +4731,7 @@ checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ "bytes", "libc", - "socket2 0.5.4", + "socket2 0.5.7", "tracing", "windows-sys 0.48.0", ] @@ -4526,6 +4851,16 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -4546,9 +4881,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -4556,9 +4891,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -4605,7 +4940,7 @@ dependencies = [ "sha1_smol", "tokio", "tokio-native-tls", - "tokio-util", + "tokio-util 0.7.9", "url", ] @@ -4627,6 +4962,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -4640,14 +4984,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.9", - "regex-syntax 0.7.5", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -4661,13 +5005,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.4", ] [[package]] @@ -4678,9 +5022,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rend" @@ -4693,12 +5037,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -4717,16 +5061,17 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", - "tokio-util", + "tokio-util 0.7.9", "tower-service", "url", "wasm-bindgen", @@ -4917,15 +5262,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.10", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.14", + "windows-sys 0.52.0", ] [[package]] @@ -4942,12 +5287,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.2", "rustls-webpki", "sct", ] @@ -4970,17 +5315,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.2", + "untrusted 0.9.0", ] [[package]] @@ -4995,6 +5340,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.22" @@ -5277,9 +5631,18 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "seqlock" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c67b6f14ecc5b86c66fa63d76b5092352678545a8a3cdae80aef5128371910" +dependencies = [ + "parking_lot 0.12.1", +] [[package]] name = "serde" @@ -5292,9 +5655,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] @@ -5323,11 +5686,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5366,19 +5729,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" dependencies = [ - "base64 0.21.4", + "base64 0.22.0", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.2", + "indexmap 2.2.6", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.6.1", + "serde_with_macros 3.8.1", "time 0.3.29", ] @@ -5396,9 +5759,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling", "proc-macro2", @@ -5412,7 +5775,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -5425,7 +5788,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ - "dashmap 5.5.3", + "dashmap", "futures", "lazy_static", "log", @@ -5562,6 +5925,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "sized-chunks" version = "0.6.5" @@ -5583,9 +5952,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -5599,12 +5968,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5625,12 +5994,12 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2" +checksum = "b94ceb26c7d19530cb1bb49bf0f817647cb5fee691dae6779e19d33ac1d4fda1" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "bv", @@ -5648,11 +6017,178 @@ dependencies = [ "zstd", ] +[[package]] +name = "solana-accounts-db" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ccab24e16797d4324444337b09555f3762b85c6a862ee5fba6a211bc6c6736" +dependencies = [ + "arrayref", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder", + "bzip2", + "crossbeam-channel", + "dashmap", + "flate2", + "fnv", + "im", + "index_list", + "itertools 0.10.5", + "lazy_static", + "log", + "lz4", + "memmap2", + "modular-bitfield", + "num-derive 0.4.1", + "num-traits", + "num_cpus", + "num_enum 0.7.2", + "ouroboros", + "percentage", + "qualifier_attr", + "rand 0.8.5", + "rayon", + "regex", + "rustc_version", + "seqlock", + "serde", + "serde_derive", + "smallvec", + "solana-bucket-map", + "solana-config-program", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-measure", + "solana-metrics", + "solana-nohash-hasher", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-vote-program", + "static_assertions", + "strum", + "strum_macros", + "tar", + "tempfile", + "thiserror", +] + +[[package]] +name = "solana-address-lookup-table-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195a93b87bd6794326a3c628cfcc8abaf70e476c4284bfb73545c873ba47c746" +dependencies = [ + "bincode", + "bytemuck", + "log", + "num-derive 0.4.1", + "num-traits", + "rustc_version", + "serde", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-program", + "solana-program-runtime", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-banks-client" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "493962c963bd1e64e2086d4b962ac87e0744bde7998eb90132e9dfdad251b8d2" +dependencies = [ + "borsh 1.5.1", + "futures", + "solana-banks-interface", + "solana-program", + "solana-sdk", + "tarpc", + "thiserror", + "tokio", + "tokio-serde", +] + +[[package]] +name = "solana-banks-interface" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d586b50a48b245369f359f043902f684abb3ad7599eff49a7387ea4641f19b" +dependencies = [ + "serde", + "solana-sdk", + "tarpc", +] + +[[package]] +name = "solana-banks-server" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee5ede19378812cf20c198a4be0c6ebf4b102d65cb0aa260f96c976030529a4" +dependencies = [ + "bincode", + "crossbeam-channel", + "futures", + "solana-accounts-db", + "solana-banks-interface", + "solana-client", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "tarpc", + "tokio", + "tokio-serde", +] + +[[package]] +name = "solana-bpf-loader-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "242706b2e7129926b929c6c5e7b9bf6659968a56bb666e7e07236167375eec99" +dependencies = [ + "bincode", + "byteorder", + "libsecp256k1", + "log", + "scopeguard", + "solana-measure", + "solana-program-runtime", + "solana-sdk", + "solana-zk-token-sdk", + "solana_rbpf", + "thiserror", +] + +[[package]] +name = "solana-bucket-map" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d6de433be2c195c9139e7cba3b13c48e8db4b135cdb571e3225c34790d2adc" +dependencies = [ + "bv", + "bytemuck", + "log", + "memmap2", + "modular-bitfield", + "num_enum 0.7.2", + "rand 0.8.5", + "solana-measure", + "solana-sdk", + "tempfile", +] + [[package]] name = "solana-clap-utils" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb19b9bbd92eee2d8f637026559a9fb48bd98aba534caedf070498a50c91fce8" +checksum = "32063b76137e13fba2f93fdf67f999d84ac587af962b48cb4115f272037b2fae" dependencies = [ "chrono", "clap 2.34.0", @@ -5667,16 +6203,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9538e3db584a8b1e70060f1f24222b8e0429f18b607f531fb45eb826f4917265" +checksum = "31f6eaf198c544d4f448e6a86a51f9af8b328f041a490a007164a75194edf341" dependencies = [ "async-trait", "bincode", - "dashmap 4.0.2", + "dashmap", "futures", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "quinn", @@ -5698,11 +6234,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-compute-budget-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a2af1d2aa3ea705147b99a34265b209393bae70930150d0583666305f91b958" +dependencies = [ + "solana-program-runtime", + "solana-sdk", +] + [[package]] name = "solana-config-program" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132" +checksum = "378259800dc9dad34828d8be4ce0de71146bac1cbbd310f8901f6f19d92c5ea3" dependencies = [ "bincode", "chrono", @@ -5714,15 +6260,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92716758e8c0e1c0bc2a5ac2eb3df443a0337fd3991cd38a3b02b12c3fbd18ce" +checksum = "58b04571089f55754f5a09493ae0bcf8d8d5c8d9cd05be3e77c502f6b68c1a3b" dependencies = [ "async-trait", "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "log", "rand 0.8.5", "rayon", @@ -5734,19 +6280,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-cost-model" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c90655f4843a771d8cb94c6601c8dfac0ef0d5465bd12b0369047c0d12fd67" +dependencies = [ + "lazy_static", + "log", + "rustc_version", + "solana-address-lookup-table-program", + "solana-bpf-loader-program", + "solana-compute-budget-program", + "solana-config-program", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-loader-v4-program", + "solana-metrics", + "solana-program-runtime", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-vote-program", +] + [[package]] name = "solana-frozen-abi" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e" +checksum = "780402262644f9efe9ac7d885812d245007fe65fd56a3dfed83ed30d61b44c74" dependencies = [ - "ahash 0.8.5", - "blake3", "block-buffer 0.10.4", "bs58 0.4.0", "bv", - "byteorder", - "cc", "either", "generic-array", "im", @@ -5757,7 +6323,6 @@ dependencies = [ "serde", "serde_bytes", "serde_derive", - "serde_json", "sha2 0.10.8", "solana-frozen-abi-macro", "subtle", @@ -5766,9 +6331,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3" +checksum = "df836de37aba77234c7afa1d857dc450fb9983572e4c6f595c84cdda65a63792" dependencies = [ "proc-macro2", "quote", @@ -5778,9 +6343,9 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-interface" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633425dc9409c6d3f019658b90bb1ad53747ed55acd45e0a18f58b95a0b89e5" +checksum = "7372ddfbe94019a1263f2c8ee16e7778778787eab6d9c8e78a52c9cb86241ea2" dependencies = [ "log", "solana-sdk", @@ -5788,11 +6353,24 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-loader-v4-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15f85c202dda9a5aaa5a3c4948a55cb9b37731dd5811198bb58ced1bbfbac40" +dependencies = [ + "log", + "solana-measure", + "solana-program-runtime", + "solana-sdk", + "solana_rbpf", +] + [[package]] name = "solana-logger" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294" +checksum = "9906be6edd0e1b579510736c153dbc51e5968808098d1b1f8c89dbea960aba58" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -5801,9 +6379,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0" +checksum = "7dc77e7f99fa5e845437ac9a593cd4bd67b5f9e4ba4a9578355eef25d3e839e9" dependencies = [ "log", "solana-sdk", @@ -5811,9 +6389,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70" +checksum = "b0a9f68887ac31f84ef69365bdc2d7ca6bf19d50a9c6ee10806adb033e24e318" dependencies = [ "crossbeam-channel", "gethostname", @@ -5826,9 +6404,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e640a95d317cad1322015c5a2b6a71697fd8dabebcb8dd33ed7f5a22869d12" +checksum = "1ba07cceff31b644df6cd4acc909df077721fa047b624b9fa906d56bcc67435a" dependencies = [ "bincode", "clap 3.2.25", @@ -5838,7 +6416,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_derive", - "socket2 0.5.4", + "socket2 0.5.7", "solana-logger", "solana-sdk", "solana-version", @@ -5846,13 +6424,19 @@ dependencies = [ "url", ] +[[package]] +name = "solana-nohash-hasher" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" + [[package]] name = "solana-perf" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266c4bd46620a925b8d508c26578d5559e97fcff6735fd22e39f369c3996ee1" +checksum = "0b7fa87b3344f96afe1395b5bb822db2ad03cdc2dbe8338636d57c58102a520b" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.11", "bincode", "bv", "caps", @@ -5877,20 +6461,21 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652" +checksum = "48ecc7af7594674687260a4d7bcfb0588cefdbe9d0f6c4e9f3140999107566c4" dependencies = [ "ark-bn254", "ark-ec", "ark-ff", "ark-serialize", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", + "bitflags 2.5.0", "blake3", "borsh 0.10.3", "borsh 0.9.3", + "borsh 1.5.1", "bs58 0.4.0", "bv", "bytemuck", @@ -5908,7 +6493,7 @@ dependencies = [ "log", "memoffset 0.9.0", "num-bigint 0.4.4", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "parking_lot 0.12.1", "rand 0.8.5", @@ -5931,18 +6516,18 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6" +checksum = "0ef9218f50470228ebca12bb147650ca7052678aad915a4e19687ee215f8d947" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bincode", "eager", "enum-iterator", "itertools 0.10.5", "libc", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "percentage", "rand 0.8.5", @@ -5957,11 +6542,41 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-program-test" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8725c449e05bec85f7ced22dc2eb88f6d45f0c4862eaebabc5fa1061fabbb1d" +dependencies = [ + "assert_matches", + "async-trait", + "base64 0.21.7", + "bincode", + "chrono-humanize", + "crossbeam-channel", + "log", + "serde", + "solana-accounts-db", + "solana-banks-client", + "solana-banks-interface", + "solana-banks-server", + "solana-bpf-loader-program", + "solana-logger", + "solana-program-runtime", + "solana-runtime", + "solana-sdk", + "solana-vote-program", + "solana_rbpf", + "test-case", + "thiserror", + "tokio", +] + [[package]] name = "solana-pubsub-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0dc2b26a7a9860f180ce11f69b0ff2a8bea0d4b9e97daee741b1e76565b3c82" +checksum = "60325aaab2bcd99ca51e1ff5a4673027a03591353a944151690b38d5dadc2c0f" dependencies = [ "crossbeam-channel", "futures-util", @@ -5984,9 +6599,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727474945d51be37ffe03e7b1d6c9630da41228c7b298a8f45098c203a78ac89" +checksum = "d979690c6c392ffdb40a91e241a08ec3942eb217bddb0403b6174de0173ab61e" dependencies = [ "async-mutex", "async-trait", @@ -5997,7 +6612,7 @@ dependencies = [ "quinn", "quinn-proto", "rcgen", - "rustls 0.21.7", + "rustls 0.21.12", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -6011,9 +6626,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "853794cccf3bd1984419a594040dfed19666e5a9ad33b0906d4174bc394b22af" +checksum = "6902079fb9d0bd4c455b97f5e48e2412d98e0e1facf635ec6fc6b783c0f3e2af" dependencies = [ "lazy_static", "num_cpus", @@ -6021,14 +6636,14 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b368f270526a5f92ec47c45a6b74ac304b62b08c169b45cf91e0d2f1703889bd" +checksum = "af88bad970c0dd63e98e7cc4c3c16a58acf32d4255aee79f611ea375592028ec" dependencies = [ "console", "dialoguer", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "parking_lot 0.12.1", "qstring", @@ -6040,12 +6655,12 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b766876b0c56950ab530d8495ef7eeaeb79e162f03dadaffc0d6852de9e844" +checksum = "d1adab0dcdc851fc7bc6ca8c6926d9f56ed3982f1e4fabd67d362647b57143d3" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bs58 0.4.0", "indicatif", @@ -6066,11 +6681,11 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876b2e410cc2403ea3216893f05034b02a180431100eb831d0b67b14fca4d29f" +checksum = "6d6764712822bbc0259bbb5413377798a11462221863d000082f39968ce5ad03" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", "reqwest", @@ -6088,9 +6703,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebdb3f02fb3cce3c967f718bc77b79433c24aa801b63dc70f374e8759b2424e4" +checksum = "49489fe59d308c10a2b3e3ecd3bee1107b9b67a325c99ffd278ba0870a5619cd" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -6099,17 +6714,94 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-runtime" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1359ea01de4d1575c7c6004a57b257da66e7b8e636e72d728c91bc6768fd593f" +dependencies = [ + "aquamarine", + "arrayref", + "base64 0.21.7", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder", + "bzip2", + "crossbeam-channel", + "dashmap", + "dir-diff", + "flate2", + "fnv", + "im", + "index_list", + "itertools 0.10.5", + "lazy_static", + "log", + "lru", + "lz4", + "memmap2", + "mockall", + "modular-bitfield", + "num-derive 0.4.1", + "num-traits", + "num_cpus", + "num_enum 0.7.2", + "ouroboros", + "percentage", + "qualifier_attr", + "rand 0.8.5", + "rayon", + "regex", + "rustc_version", + "serde", + "serde_derive", + "serde_json", + "solana-accounts-db", + "solana-address-lookup-table-program", + "solana-bpf-loader-program", + "solana-bucket-map", + "solana-compute-budget-program", + "solana-config-program", + "solana-cost-model", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-loader-v4-program", + "solana-measure", + "solana-metrics", + "solana-perf", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-version", + "solana-vote", + "solana-vote-program", + "solana-zk-token-proof-program", + "solana-zk-token-sdk", + "static_assertions", + "strum", + "strum_macros", + "symlink", + "tar", + "tempfile", + "thiserror", + "zstd", +] + [[package]] name = "solana-sdk" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad" +checksum = "73bb113fa17e0607343afdc795c2c5230981c5b51c99b2c54fba91755879d65b" dependencies = [ "assert_matches", - "base64 0.21.4", + "base64 0.21.7", "bincode", - "bitflags 2.4.0", - "borsh 0.10.3", + "bitflags 2.5.0", + "borsh 1.5.1", "bs58 0.4.0", "bytemuck", "byteorder", @@ -6126,9 +6818,9 @@ dependencies = [ "libsecp256k1", "log", "memmap2", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", - "num_enum 0.6.1", + "num_enum 0.7.2", "pbkdf2 0.11.0", "qstring", "qualifier_attr", @@ -6143,6 +6835,7 @@ dependencies = [ "serde_with 2.3.3", "sha2 0.10.8", "sha3 0.10.8", + "siphasher", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-logger", @@ -6155,9 +6848,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784" +checksum = "8fcdb3a94e2f04d856d2fba6feb4f6887a1da21a3ee0b64b69c08d15dc22d46c" dependencies = [ "bs58 0.4.0", "proc-macro2", @@ -6172,18 +6865,49 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" +[[package]] +name = "solana-send-transaction-service" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7550c870a6648614b0a34dd3ebf69d98a8b80ab808050aac207e17fe77e8b2d8" +dependencies = [ + "crossbeam-channel", + "log", + "solana-client", + "solana-measure", + "solana-metrics", + "solana-runtime", + "solana-sdk", + "solana-tpu-client", +] + +[[package]] +name = "solana-stake-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfb5f94949dbd8839cfe5cefd38b077a395a64407fa2e339c38657295e0fede" +dependencies = [ + "bincode", + "log", + "rustc_version", + "solana-config-program", + "solana-program-runtime", + "solana-sdk", + "solana-vote-program", +] + [[package]] name = "solana-streamer" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70eda40efb5bc57ad50b1ac8452485065c1adae0e701a0348b397db054e2ab5" +checksum = "c0e6132c9eefb372202e69e654e8c2a30b4c06635343dd6474467b9cca4b9dd9" dependencies = [ "async-channel 1.9.0", "bytes", "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.0.2", + "indexmap 2.2.6", "itertools 0.10.5", "libc", "log", @@ -6195,20 +6919,35 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rcgen", - "rustls 0.21.7", + "rustls 0.21.12", + "smallvec", "solana-metrics", "solana-perf", "solana-sdk", - "thiserror", - "tokio", - "x509-parser", + "thiserror", + "tokio", + "x509-parser", +] + +[[package]] +name = "solana-system-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1338d45734fb10d4d378ccc84b8ff779111a7371719e92d045bcdf60db6c6ec4" +dependencies = [ + "bincode", + "log", + "serde", + "serde_derive", + "solana-program-runtime", + "solana-sdk", ] [[package]] name = "solana-thin-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3c510144695c3d1ee1f84dd9975af7f7d35c168447c484bbd35c21e903c515" +checksum = "ca1c3ed492f61914aaa8074cf7a07f93bfd8d9adbf9846939e589b7b7c70fe39" dependencies = [ "bincode", "log", @@ -6221,14 +6960,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f27c8fec609179a7dfc287060df2a926c8cd89329235c4b8d78bd019a72462" +checksum = "c3fbfce5d27608f4853dcb0c0d964f59420710a7a4486409e7583717c610c3cf" dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.0.2", + "indexmap 2.2.6", "indicatif", "log", "rayon", @@ -6245,12 +6984,12 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e" +checksum = "6adbd8f3fccddeae87278a105dcf8a8792f8816c0f4fb5f7ae8f307af279ac49" dependencies = [ "Inflector", - "base64 0.21.4", + "base64 0.21.7", "bincode", "borsh 0.10.3", "bs58 0.4.0", @@ -6270,9 +7009,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ead118c5d549e4345dc59cbc5d9b282164f3e5334707f186e3aa10d40e3b30" +checksum = "b90f8ebd26cac3cd563bf839ff8511f27698f2d220e58f7538b5d6d80d8970ed" dependencies = [ "async-trait", "solana-connection-cache", @@ -6285,9 +7024,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532f5d631562587facc5fe88abd2e31c0d1f29012b6766c664db9f05a39fb05b" +checksum = "ac8714cf9f6caefc403e19770ad73ed2e4c866b7201e31dd17a9e06b6a693a57" dependencies = [ "log", "rustc_version", @@ -6299,15 +7038,34 @@ dependencies = [ "solana-sdk", ] +[[package]] +name = "solana-vote" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db24fb82cba851f6db063400db08405a2108f68b27c3d5f2d3ed1da9849734f" +dependencies = [ + "crossbeam-channel", + "itertools 0.10.5", + "log", + "rustc_version", + "serde", + "serde_derive", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-sdk", + "solana-vote-program", + "thiserror", +] + [[package]] name = "solana-vote-program" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c684430058b0a2e733936a8851c8843a3a6316ccd5c969d39411a479d6489642" +checksum = "b5de2428939c6e279901d4357bf02c809739e5b97164e8620e09a9e0b55c2327" dependencies = [ "bincode", "log", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "rustc_version", "serde", @@ -6321,14 +7079,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-zk-token-proof-program" +version = "1.18.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d381b5205bd4ff1edfbb910895f48d00a235a0257493462ff0d102004bc337" +dependencies = [ + "bytemuck", + "num-derive 0.4.1", + "num-traits", + "solana-program-runtime", + "solana-sdk", + "solana-zk-token-sdk", +] + [[package]] name = "solana-zk-token-sdk" -version = "1.17.28" +version = "1.18.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c" +checksum = "8dad1753ec3b189879c8756ac35471467116dfc93d7aeb68cfd28f22a02c850d" dependencies = [ "aes-gcm-siv", - "base64 0.21.4", + "base64 0.21.7", "bincode", "bytemuck", "byteorder", @@ -6337,7 +7109,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "merlin", - "num-derive 0.3.3", + "num-derive 0.4.1", "num-traits", "rand 0.7.3", "serde", @@ -6394,12 +7166,11 @@ dependencies = [ [[package]] name = "spl-account-compression" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c43bd4455d9fb29b9e4f83c087ccffa2f6f41fecfc0549932ae391d00f3378" dependencies = [ "anchor-lang", "bytemuck", - "spl-concurrent-merkle-tree", + "solana-program", + "spl-concurrent-merkle-tree 0.2.0", "spl-noop", ] @@ -6419,6 +7190,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "spl-concurrent-merkle-tree" +version = "0.2.0" +dependencies = [ + "bytemuck", + "rand 0.8.5", + "rand_distr", + "solana-program", + "spl-merkle-tree-reference 0.1.0", + "thiserror", + "tokio", +] + [[package]] name = "spl-concurrent-merkle-tree" version = "0.2.0" @@ -6474,11 +7258,27 @@ dependencies = [ "solana-program", ] +[[package]] +name = "spl-merkle-tree-reference" +version = "0.1.0" +dependencies = [ + "solana-program", + "thiserror", +] + +[[package]] +name = "spl-merkle-tree-reference" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28437c617c7f0db6b7229a489239f3ea6160499542d9367fbca2fc5ec7744abb" +dependencies = [ + "solana-program", + "thiserror", +] + [[package]] name = "spl-noop" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd67ea3d0070a12ff141f5da46f9695f49384a03bce1203a5608f5739437950" dependencies = [ "solana-program", ] @@ -6489,7 +7289,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "borsh 0.10.3", "bytemuck", "serde", @@ -6762,7 +7562,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70a313e115c2cd9a88d99d60386bc88641c853d468b2c3bc454c294f385fc084" dependencies = [ - "async-channel 2.1.0", + "async-channel 2.3.1", "async-io 2.3.2", "atomic", "crossbeam-channel", @@ -6800,12 +7600,40 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "subtle" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -6828,6 +7656,24 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -6867,17 +7713,62 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tarpc" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38a012bed6fb9681d3bf71ffaa4f88f3b4b9ed3198cda6e4c8462d24d4bb80" +dependencies = [ + "anyhow", + "fnv", + "futures", + "humantime", + "opentelemetry", + "pin-project", + "rand 0.8.5", + "serde", + "static_assertions", + "tarpc-plugins", + "thiserror", + "tokio", + "tokio-serde", + "tokio-util 0.6.10", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "tarpc-plugins" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.18", - "windows-sys 0.48.0", + "rustix 0.38.34", + "windows-sys 0.52.0", ] [[package]] @@ -6889,6 +7780,45 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "test-case-core", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -6906,18 +7836,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", @@ -7009,9 +7939,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -7021,7 +7951,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.7", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -7029,9 +7959,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -7065,10 +7995,26 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.12", "tokio", ] +[[package]] +name = "tokio-serde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", + "serde_json", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -7088,13 +8034,28 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.7", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", "tungstenite", "webpki-roots 0.25.2", ] +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.9" @@ -7146,7 +8107,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -7157,7 +8118,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", @@ -7250,6 +8211,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -7339,7 +8313,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.7", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -7490,9 +8464,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -7563,6 +8537,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -7598,9 +8582,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7608,9 +8592,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -7635,9 +8619,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7645,9 +8629,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -7658,9 +8642,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" @@ -7769,15 +8753,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -7796,21 +8771,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -7842,12 +8802,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.5", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -7860,12 +8814,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -7878,12 +8826,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -7902,12 +8844,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -7920,12 +8856,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -7938,12 +8868,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -7956,12 +8880,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -8020,11 +8938,22 @@ dependencies = [ "time 0.3.29", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys 0.4.14", + "rustix 0.38.34", +] + [[package]] name = "xxhash-rust" -version = "0.8.7" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" +checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" [[package]] name = "yaml-rust" diff --git a/Cargo.toml b/Cargo.toml index 98d55ecff..01848e610 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ - "blockbuster", + "blockbuster/blockbuster", "core", "das_api", "digital_asset_types", @@ -32,7 +32,7 @@ anyhow = "1.0.75" async-std = "1.0.0" async-trait = "0.1.60" backon = "0.4.1" -blockbuster = {path = "blockbuster"} +blockbuster = {path = "blockbuster/blockbuster"} borsh = "~0.10.3" borsh-derive = "~0.10.3" bs58 = "0.4.0" @@ -67,8 +67,8 @@ log = "0.4.17" metrics = "0.20.1" migration = {path = "migration"} mime_guess = "2.0.4" -mpl-bubblegum = "1.2.0" -mpl-core = {version = "0.7.0", features = ["serde"]} +mpl-bubblegum = { git = "https://github.com/adm-metaex/mpl-bubblegum.git", branch = "feature/cleanup", features = ["serde"] } +mpl-core = { git = "https://github.com/RequescoS/mpl-core.git", features = ["serde"] } mpl-token-metadata = "4.1.1" nft_ingester = {path = "nft_ingester"} num-derive = "0.3.3" @@ -77,7 +77,7 @@ once_cell = "1.19.0" open-rpc-derive = "0.0.4" open-rpc-schema = "0.0.4" plerkle_messenger = "1.6.0" -plerkle_serialization = "1.8.0" +plerkle_serialization = { git = "https://github.com/n00m4d/digital-asset-validator-plugin.git" } program_transformers = {path = "program_transformers"} prometheus = "0.13.3" proxy-wasm = "0.2.0" @@ -93,17 +93,17 @@ sea-query = "0.28.1" serde = "1.0.137" serde_json = "1.0.81" serial_test = "2.0.0" -solana-account-decoder = "~1.17" -solana-client = "~1.17" -solana-geyser-plugin-interface = "~1.17" -solana-program = "~1.17" -solana-sdk = "~1.17" -solana-transaction-status = "~1.17" +solana-account-decoder = "1.18.11" +solana-client = "1.18.11" +solana-geyser-plugin-interface = "1.18.11" +solana-program = "1.18.11" +solana-sdk = "1.18.11" +solana-transaction-status = "1.18.11" solana-zk-token-sdk = "1.17.16" -spl-account-compression = "0.3.0" +spl-account-compression = { git = "https://github.com/StanChe/solana-program-library.git", branch = "feature/init_with_root", features = ["no-entrypoint"] } spl-associated-token-account = ">= 1.1.3, < 3.0" -spl-concurrent-merkle-tree = "0.2.0" -spl-noop = "0.2.0" +spl-concurrent-merkle-tree = { git = "https://github.com/StanChe/solana-program-library.git", branch = "feature/init_with_root" } +spl-noop = { git = "https://github.com/StanChe/solana-program-library.git", branch = "feature/init_with_root", features = ["no-entrypoint"] } spl-pod = {version = "0.1.0", features = ["serde-traits"]} spl-token = ">= 3.5.0, < 5.0" spl-token-2022 = {version = "1.0", features = ["no-entrypoint"]} @@ -111,7 +111,7 @@ spl-token-group-interface = "0.1.0" spl-token-metadata-interface = "0.2.0" sqlx = "0.6.2" stretto = "0.8.4" -thiserror = "1.0.31" +thiserror = "1.0.63" tokio = "1.30.0" tokio-stream = "0.1.14" tower = "0.4.13" @@ -122,6 +122,16 @@ txn_forwarder = {path = "tools/txn_forwarder"} url = "2.3.1" wasi = "0.7.0" wasm-bindgen = "0.2.83" +triomphe = "=0.1.9" +paste = "1" +serde_derive = "1.0.190" +mockall = "0.11.4" +xxhash-rust = { version = "0.8.10", features = ["xxh3"] } +serde_with = "3.8.1" +bincode = "1.3.3" +tempfile = "3.10.1" +async-channel = "2.3.1" +bubblegum-batch-sdk = { git = "https://github.com/metaplex-foundation/bubblegum-batch-sdk.git", branch = "main" } [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/blockbuster/.gitignore b/blockbuster/.gitignore deleted file mode 100644 index 28aaccd5c..000000000 --- a/blockbuster/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Generated by Cargo -# will have compiled files and executables -/target/ - - -# These are backup files generated by rustfmt -**/*.rs.bk - -*.iml -blockbuster/target -target/ diff --git a/blockbuster/Cargo.toml b/blockbuster/Cargo.toml deleted file mode 100644 index e110009ea..000000000 --- a/blockbuster/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -authors = ["Metaplex Developers "] -description = "Metaplex canonical program parsers, for indexing, analytics etc...." -edition = "2021" -license = "AGPL-3.0" -name = "blockbuster" -readme = "../README.md" -repository = "https://github.com/metaplex-foundation/blockbuster" -version = "2.3.0" - -[dependencies] -anchor-lang = {workspace = true} -async-trait = {workspace = true} -borsh = {workspace = true} -bs58 = {workspace = true} -bytemuck = {workspace = true} -lazy_static = {workspace = true} -log = {workspace = true} -mpl-bubblegum = {workspace = true} -mpl-core = {workspace = true, features = ["serde"]} -mpl-token-metadata = {workspace = true, features = ["serde"]} -serde = {workspace = true} -solana-sdk = {workspace = true} -solana-transaction-status = {workspace = true} -solana-zk-token-sdk = {workspace = true} -spl-account-compression = {workspace = true, features = ["no-entrypoint"]} -spl-noop = {workspace = true, features = ["no-entrypoint"]} -spl-pod = {workspace = true, features = ["serde-traits"]} -spl-token = {workspace = true, features = ["no-entrypoint"]} -spl-token-2022 = {workspace = true, features = ["no-entrypoint"]} -spl-token-group-interface = {workspace = true} -spl-token-metadata-interface = {workspace = true} -thiserror = {workspace = true} - -[dev-dependencies] -flatbuffers = {workspace = true} -plerkle_serialization = {workspace = true} -rand = {workspace = true} -serde_json = {workspace = true} -solana-client = {workspace = true} -solana-geyser-plugin-interface = {workspace = true} -spl-concurrent-merkle-tree = {workspace = true} diff --git a/blockbuster/README.md b/blockbuster/README.md deleted file mode 100644 index 896d8bbc5..000000000 --- a/blockbuster/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# BlockBuster - -BlockBuster -> "Busting Solana blocks into little pieces to index and operate on the programs therein" - Noone - 1995 - -This repository is the home for Metaplex Program Parsers. Program parsers are canonical libraries that take a transaction or account update from a geyser plugin and parse them correctly according to Metaplex smart contracts. This sort of parsing is hard to automate as it must contain some knowledge of the API structure of the contract which is not fully describable yet via IDLs. Things like remaining accounts, optional accounts and complex instruction data are not always 100% clear what they mean without knowledge of the contract. - -## Mode of Operation -This library works best as a consumer of messages sent via a geyser plugin using the [Plerkle Serialization](https://github.com/metaplex-foundation/digital-asset-validator-plugin) library by metaplex. The types from that library are FlatBuffer based currently, and are the wire format of messages coming out of Plerkle into the rest of the infrastructure. -For more information about Plerkle and the [Digital Asset RPC infrastructure](https://github.com/metaplex-foundation/digital-asset-validator-plugin) It can however be used in any general programs provided you can create the data in the FlatBuffer types. - -## Scope - -This library contains parsers for the following programs and the parsers are specific to how these contracts relate to metaplex assets. - -* Gummyroll (Solana) -* Bubblegum (Metaplex) -* Spl Token (Solana) -* Token Metadata (Metaplex) -* Auction House (Metaplex) -* Candy Machine (Metaplex) -* Hydra (Metaplex) - diff --git a/blockbuster/blockbuster/Cargo.toml b/blockbuster/blockbuster/Cargo.toml new file mode 100644 index 000000000..c385ed6a9 --- /dev/null +++ b/blockbuster/blockbuster/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "blockbuster" +description = "Metaplex canonical program parsers, for indexing, analytics etc...." +version = "2.3.0" +authors = ["Metaplex Developers "] +repository = "https://github.com/metaplex-foundation/blockbuster" +license = "AGPL-3.0" +edition = "2021" +readme = "../README.md" + +[dependencies] +bytemuck = { version = "1.14.0", features = ["derive"] } +spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] } +spl-account-compression = { git = "https://github.com/StanChe/solana-program-library.git", branch = "feature/init_with_root", features = ["no-entrypoint"] } +spl-noop = { workspace = true } +mpl-bubblegum = { workspace = true } +mpl-core = { git = "https://github.com/RequescoS/mpl-core.git", features = ["serde"] } +mpl-token-metadata = { version = "4.1.1", features = ["serde"] } +spl-token = { version = "4.0.0", features = ["no-entrypoint"] } +async-trait = "0.1.57" +bs58 = "0.4.0" +lazy_static = "1.4.0" +borsh = "~0.10.3" +thiserror = "1.0.63" +log = "0.4.17" +solana-sdk = "~1.18.11" +solana-transaction-status = "~1.18.11" +spl-token-metadata-interface = "0.2.0" +spl-token-group-interface = "0.1.0" +spl-pod = { version = "0.1.0", features = ["serde-traits"] } +serde = "1.0.140" +solana-zk-token-sdk = "~1.18.11" +anchor-lang = { version = "0.29.0" } +bubblegum-batch-sdk = { workspace = true } + +[dev-dependencies] +flatbuffers = "23.1.21" +plerkle_serialization = { git = "https://github.com/n00m4d/digital-asset-validator-plugin.git" } +rand = "0.8.5" +serde_json = "1.0.89" +solana-client = "~1.18.11" +solana-geyser-plugin-interface = "~1.18.11" +spl-concurrent-merkle-tree = "0.2.0" diff --git a/blockbuster/blockbuster/rustfmt.toml b/blockbuster/blockbuster/rustfmt.toml new file mode 100644 index 000000000..25c9650ab --- /dev/null +++ b/blockbuster/blockbuster/rustfmt.toml @@ -0,0 +1,3 @@ +edition = "2021" +imports_granularity="Crate" +reorder_imports = true diff --git a/blockbuster/src/error.rs b/blockbuster/blockbuster/src/error.rs similarity index 100% rename from blockbuster/src/error.rs rename to blockbuster/blockbuster/src/error.rs diff --git a/blockbuster/src/instruction.rs b/blockbuster/blockbuster/src/instruction.rs similarity index 82% rename from blockbuster/src/instruction.rs rename to blockbuster/blockbuster/src/instruction.rs index 4345e29ed..f4544c165 100644 --- a/blockbuster/src/instruction.rs +++ b/blockbuster/blockbuster/src/instruction.rs @@ -39,9 +39,7 @@ pub fn order_instructions<'a>( for (outer_instruction_index, message_instruction) in message_instructions.iter().enumerate() { let non_hoisted_inner_instruction = meta_inner_instructions .iter() - .filter_map(|ix| { - (ix.index == outer_instruction_index as u8).then_some(&ix.instructions) - }) + .filter_map(|ix| (ix.index == outer_instruction_index as u8).then(|| &ix.instructions)) .flatten() .map(|inner_ix| { let cix = &inner_ix.instruction; @@ -75,15 +73,16 @@ fn hoist_known_programs<'a>( ix_pairs .iter() .enumerate() - .filter(|&(_index, &(pid, _ci))| programs.contains(&pid)) - .map(|(index, &(pid, ci))| { - let inner_copy = ix_pairs - .iter() - .skip(index + 1) - .take_while(|&&(inner_pid, _)| inner_pid != pid) - .cloned() - .collect::>>(); - ((pid, ci), Some(inner_copy)) + .filter_map(|(index, &(pid, ci))| { + programs.contains(&pid).then(|| { + let inner_copy = ix_pairs + .iter() + .skip(index + 1) + .take_while(|&&(inner_pid, _)| inner_pid != pid) + .cloned() + .collect::>>(); + ((pid, ci), Some(inner_copy)) + }) }) .collect() } diff --git a/blockbuster/src/lib.rs b/blockbuster/blockbuster/src/lib.rs similarity index 100% rename from blockbuster/src/lib.rs rename to blockbuster/blockbuster/src/lib.rs diff --git a/blockbuster/src/parsed_programs.rs b/blockbuster/blockbuster/src/parsed_programs.rs similarity index 100% rename from blockbuster/src/parsed_programs.rs rename to blockbuster/blockbuster/src/parsed_programs.rs diff --git a/blockbuster/src/program_handler.rs b/blockbuster/blockbuster/src/program_handler.rs similarity index 100% rename from blockbuster/src/program_handler.rs rename to blockbuster/blockbuster/src/program_handler.rs diff --git a/blockbuster/src/programs/bubblegum/mod.rs b/blockbuster/blockbuster/src/programs/bubblegum/mod.rs similarity index 69% rename from blockbuster/src/programs/bubblegum/mod.rs rename to blockbuster/blockbuster/src/programs/bubblegum/mod.rs index faa1b8aa1..582becdbf 100644 --- a/blockbuster/src/programs/bubblegum/mod.rs +++ b/blockbuster/blockbuster/src/programs/bubblegum/mod.rs @@ -4,14 +4,17 @@ use crate::{ program_handler::{NotUsed, ParseResult, ProgramParser}, programs::ProgramParseResult, }; -use borsh::de::BorshDeserialize; +use borsh::{BorshDeserialize, BorshSerialize}; +use bubblegum_batch_sdk::model::BatchMintInstruction; use log::warn; use mpl_bubblegum::{ get_instruction_type, instructions::{ - UnverifyCreatorInstructionArgs, UpdateMetadataInstructionArgs, VerifyCreatorInstructionArgs, + FinalizeTreeWithRootAndCollectionInstructionArgs, FinalizeTreeWithRootInstructionArgs, + UnverifyCreatorInstructionArgs, UpdateMetadataInstructionArgs, + VerifyCreatorInstructionArgs, }, - types::{BubblegumEventType, MetadataArgs, UpdateArgs}, + types::{BubblegumEventType, MetadataArgs, UpdateArgs, Version}, }; pub use mpl_bubblegum::{ types::{LeafSchema, UseMethod}, @@ -25,6 +28,35 @@ pub use spl_account_compression::events::{ use spl_noop; +#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct FinalizeTreeWithRootInstructionArgsWithStaker { + pub root: [u8; 32], + pub rightmost_leaf: [u8; 32], + pub rightmost_index: u32, + pub metadata_url: String, + pub metadata_hash: String, + pub staker: Pubkey, + pub collection_mint: Option, +} +impl FinalizeTreeWithRootInstructionArgsWithStaker { + fn build_finalize_tree_with_root_instruction_args_with_staker( + args: FinalizeTreeWithRootInstructionArgs, + staker: Pubkey, + collection_mint: Option, + ) -> Self { + Self { + root: args.root, + rightmost_leaf: args.rightmost_leaf, + rightmost_index: args.rightmost_index, + metadata_url: args.metadata_url, + metadata_hash: args.metadata_hash, + staker, + collection_mint, + } + } +} + #[derive(Eq, PartialEq)] pub enum Payload { Unknown, @@ -53,6 +85,10 @@ pub enum Payload { update_args: UpdateArgs, tree_id: Pubkey, }, + FinalizeTreeWithRoot { + args: FinalizeTreeWithRootInstructionArgsWithStaker, + tree_id: Pubkey, + }, } //TODO add more of the parsing here to minimize program transformer code pub struct BubblegumInstruction { @@ -62,6 +98,26 @@ pub struct BubblegumInstruction { pub payload: Option, } +impl From<&BatchMintInstruction> for BubblegumInstruction { + fn from(value: &BatchMintInstruction) -> Self { + let hash = value.leaf_update.hash(); + Self { + instruction: InstructionName::MintV1, + tree_update: Some((&value.tree_update).into()), + leaf_update: Some(LeafSchemaEvent::new( + Version::V1, + value.leaf_update.clone(), + hash, + )), + payload: Some(Payload::MintV1 { + args: value.mint_args.clone(), + authority: value.authority, + tree_id: value.tree_update.id, + }), + } + } +} + impl BubblegumInstruction { pub fn new(ix: InstructionName) -> Self { BubblegumInstruction { @@ -206,6 +262,14 @@ impl ProgramParser for BubblegumParser { InstructionName::UpdateMetadata => { b_inst.payload = Some(build_update_metadata_payload(keys, ix_data)?); } + InstructionName::FinalizeTreeWithRoot => { + b_inst.payload = Some(build_create_tree_with_root_payload(keys, ix_data)?); + } + InstructionName::FinalizeTreeWithRootAndCollection => { + b_inst.payload = Some(build_create_tree_with_root_and_collection_payload( + keys, ix_data, + )?); + } _ => {} }; } @@ -299,3 +363,50 @@ fn build_update_metadata_payload( tree_id, }) } + +// See Bubblegum for offsets and positions: +// https://github.com/metaplex-foundation/mpl-bubblegum/blob/main/programs/bubblegum/README.md +fn build_create_tree_with_root_payload( + keys: &[Pubkey], + ix_data: &[u8], +) -> Result { + let args = FinalizeTreeWithRootInstructionArgs::try_from_slice(ix_data)?; + + let tree_id = *keys + .get(1) + .ok_or(BlockbusterError::InstructionParsingError)?; + let staker = *keys + .get(4) + .ok_or(BlockbusterError::InstructionParsingError)?; + let args = FinalizeTreeWithRootInstructionArgsWithStaker::build_finalize_tree_with_root_instruction_args_with_staker(args, staker, None); + + Ok(Payload::FinalizeTreeWithRoot { args, tree_id }) +} + +// See Bubblegum for offsets and positions: +// https://github.com/metaplex-foundation/mpl-bubblegum/blob/main/programs/bubblegum/README.md +fn build_create_tree_with_root_and_collection_payload( + keys: &[Pubkey], + ix_data: &[u8], +) -> Result { + let args = FinalizeTreeWithRootAndCollectionInstructionArgs::try_from_slice(ix_data)?; + + let tree_id = *keys + .get(1) + .ok_or(BlockbusterError::InstructionParsingError)?; + let staker = *keys + .get(4) + .ok_or(BlockbusterError::InstructionParsingError)?; + let collection_mint = *keys + .get(11) + .ok_or(BlockbusterError::InstructionParsingError)?; + let args = FinalizeTreeWithRootInstructionArgsWithStaker::build_finalize_tree_with_root_instruction_args_with_staker(FinalizeTreeWithRootInstructionArgs { + root: args.root, + rightmost_leaf: args.rightmost_leaf, + rightmost_index: args.rightmost_index, + metadata_url: args.metadata_url, + metadata_hash: args.metadata_hash, + }, staker, Some(collection_mint)); + + Ok(Payload::FinalizeTreeWithRoot { args, tree_id }) +} diff --git a/blockbuster/src/programs/mod.rs b/blockbuster/blockbuster/src/programs/mod.rs similarity index 100% rename from blockbuster/src/programs/mod.rs rename to blockbuster/blockbuster/src/programs/mod.rs diff --git a/blockbuster/src/programs/mpl_core_program/mod.rs b/blockbuster/blockbuster/src/programs/mpl_core_program/mod.rs similarity index 100% rename from blockbuster/src/programs/mpl_core_program/mod.rs rename to blockbuster/blockbuster/src/programs/mpl_core_program/mod.rs diff --git a/blockbuster/src/programs/token_account/mod.rs b/blockbuster/blockbuster/src/programs/token_account/mod.rs similarity index 100% rename from blockbuster/src/programs/token_account/mod.rs rename to blockbuster/blockbuster/src/programs/token_account/mod.rs diff --git a/blockbuster/src/programs/token_extensions/extension.rs b/blockbuster/blockbuster/src/programs/token_extensions/extension.rs similarity index 100% rename from blockbuster/src/programs/token_extensions/extension.rs rename to blockbuster/blockbuster/src/programs/token_extensions/extension.rs diff --git a/blockbuster/src/programs/token_extensions/mod.rs b/blockbuster/blockbuster/src/programs/token_extensions/mod.rs similarity index 100% rename from blockbuster/src/programs/token_extensions/mod.rs rename to blockbuster/blockbuster/src/programs/token_extensions/mod.rs diff --git a/blockbuster/src/programs/token_metadata/mod.rs b/blockbuster/blockbuster/src/programs/token_metadata/mod.rs similarity index 100% rename from blockbuster/src/programs/token_metadata/mod.rs rename to blockbuster/blockbuster/src/programs/token_metadata/mod.rs diff --git a/blockbuster/tests/bubblegum_parser_test.rs b/blockbuster/blockbuster/tests/bubblegum_parser_test.rs similarity index 100% rename from blockbuster/tests/bubblegum_parser_test.rs rename to blockbuster/blockbuster/tests/bubblegum_parser_test.rs diff --git a/blockbuster/tests/fixtures/double_bubblegum_mint.json b/blockbuster/blockbuster/tests/fixtures/double_bubblegum_mint.json similarity index 100% rename from blockbuster/tests/fixtures/double_bubblegum_mint.json rename to blockbuster/blockbuster/tests/fixtures/double_bubblegum_mint.json diff --git a/blockbuster/tests/fixtures/helium_mint_double_tree.json b/blockbuster/blockbuster/tests/fixtures/helium_mint_double_tree.json similarity index 100% rename from blockbuster/tests/fixtures/helium_mint_double_tree.json rename to blockbuster/blockbuster/tests/fixtures/helium_mint_double_tree.json diff --git a/blockbuster/tests/fixtures/helium_nested.json b/blockbuster/blockbuster/tests/fixtures/helium_nested.json similarity index 100% rename from blockbuster/tests/fixtures/helium_nested.json rename to blockbuster/blockbuster/tests/fixtures/helium_nested.json diff --git a/blockbuster/tests/helpers.rs b/blockbuster/blockbuster/tests/helpers.rs similarity index 100% rename from blockbuster/tests/helpers.rs rename to blockbuster/blockbuster/tests/helpers.rs diff --git a/blockbuster/tests/instructions_test.rs b/blockbuster/blockbuster/tests/instructions_test.rs similarity index 100% rename from blockbuster/tests/instructions_test.rs rename to blockbuster/blockbuster/tests/instructions_test.rs diff --git a/blockbuster/rust-toolchain.toml b/blockbuster/rust-toolchain.toml new file mode 100644 index 000000000..4dd8e5c56 --- /dev/null +++ b/blockbuster/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.75.0" \ No newline at end of file diff --git a/blockbuster/src/programs/account_closure/mod.rs b/blockbuster/src/programs/account_closure/mod.rs deleted file mode 100644 index 7586b7d22..000000000 --- a/blockbuster/src/programs/account_closure/mod.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::{ - error::BlockbusterError, - program_handler::{ParseResult, ProgramParser}, - programs::ProgramParseResult, -}; -use solana_sdk::{pubkey::Pubkey, pubkeys}; - -use plerkle_serialization::AccountInfo; - -pubkeys!(solana_program_id, "11111111111111111111111111111111"); - -pub struct ClosedAccountInfo { - pub pubkey: Vec, - pub owner: Vec, -} - -#[allow(clippy::large_enum_variant)] -pub enum AccountClosureData { - ClosedAccountInfo(ClosedAccountInfo), - EmptyAccount, -} - -impl ParseResult for AccountClosureData { - fn result(&self) -> &Self - where - Self: Sized, - { - self - } - fn result_type(&self) -> ProgramParseResult { - ProgramParseResult::AccountClosure(self) - } -} - -pub struct AccountClosureParser; - -impl ProgramParser for AccountClosureParser { - fn key(&self) -> Pubkey { - solana_program_id() - } - fn key_match(&self, key: &Pubkey) -> bool { - key == &solana_program_id() - } - - fn handles_account_updates(&self) -> bool { - true - } - - fn handles_instructions(&self) -> bool { - false - } - - fn handle_account( - &self, - account_info: &AccountInfo, - ) -> Result, BlockbusterError> { - let account_data: ClosedAccountInfo = match (account_info.pubkey(), account_info.owner()) { - (Some(pubkey), Some(owner)) => ClosedAccountInfo { - pubkey: pubkey.0.to_vec(), - owner: owner.0.to_vec(), - }, - _ => return Ok(Box::new(AccountClosureData::EmptyAccount)), - }; - - if account_info.lamports() == 0 { - Ok(Box::new(AccountClosureData::ClosedAccountInfo( - account_data, - ))) - } else { - Ok(Box::new(AccountClosureData::EmptyAccount)) - } - } -} diff --git a/digital_asset_types/src/dao/generated/asset.rs b/digital_asset_types/src/dao/generated/asset.rs index e4f4ab0ac..12ae5ec49 100644 --- a/digital_asset_types/src/dao/generated/asset.rs +++ b/digital_asset_types/src/dao/generated/asset.rs @@ -41,15 +41,15 @@ pub struct Model { pub created_at: Option, pub burnt: bool, pub slot_updated: Option, - pub slot_updated_metadata_account: Option, - pub slot_updated_mint_account: Option, - pub slot_updated_token_account: Option, - pub slot_updated_cnft_transaction: Option, pub data_hash: Option, pub creator_hash: Option, pub owner_delegate_seq: Option, pub leaf_seq: Option, pub base_info_seq: Option, + pub slot_updated_metadata_account: Option, + pub slot_updated_token_account: Option, + pub slot_updated_mint_account: Option, + pub slot_updated_cnft_transaction: Option, pub mpl_core_plugins: Option, pub mpl_core_unknown_plugins: Option, pub mpl_core_collection_num_minted: Option, @@ -84,15 +84,15 @@ pub enum Column { CreatedAt, Burnt, SlotUpdated, - SlotUpdatedMetadataAccount, - SlotUpdatedMintAccount, - SlotUpdatedTokenAccount, - SlotUpdatedCnftTransaction, DataHash, CreatorHash, OwnerDelegateSeq, LeafSeq, BaseInfoSeq, + SlotUpdatedMetadataAccount, + SlotUpdatedTokenAccount, + SlotUpdatedMintAccount, + SlotUpdatedCnftTransaction, MplCorePlugins, MplCoreUnknownPlugins, MplCoreCollectionNumMinted, @@ -144,15 +144,15 @@ impl ColumnTrait for Column { Self::CreatedAt => ColumnType::TimestampWithTimeZone.def().null(), Self::Burnt => ColumnType::Boolean.def(), Self::SlotUpdated => ColumnType::BigInteger.def().null(), - Self::SlotUpdatedMetadataAccount => ColumnType::BigInteger.def().null(), - Self::SlotUpdatedMintAccount => ColumnType::BigInteger.def().null(), - Self::SlotUpdatedTokenAccount => ColumnType::BigInteger.def().null(), - Self::SlotUpdatedCnftTransaction => ColumnType::BigInteger.def().null(), Self::DataHash => ColumnType::Char(Some(50u32)).def().null(), Self::CreatorHash => ColumnType::Char(Some(50u32)).def().null(), Self::OwnerDelegateSeq => ColumnType::BigInteger.def().null(), Self::LeafSeq => ColumnType::BigInteger.def().null(), Self::BaseInfoSeq => ColumnType::BigInteger.def().null(), + Self::SlotUpdatedMetadataAccount => ColumnType::BigInteger.def().null(), + Self::SlotUpdatedTokenAccount => ColumnType::BigInteger.def().null(), + Self::SlotUpdatedMintAccount => ColumnType::BigInteger.def().null(), + Self::SlotUpdatedCnftTransaction => ColumnType::BigInteger.def().null(), Self::MplCorePlugins => ColumnType::JsonBinary.def().null(), Self::MplCoreUnknownPlugins => ColumnType::JsonBinary.def().null(), Self::MplCoreCollectionNumMinted => ColumnType::Integer.def().null(), diff --git a/digital_asset_types/src/dao/generated/batch_mint.rs b/digital_asset_types/src/dao/generated/batch_mint.rs new file mode 100644 index 000000000..8f9dc809e --- /dev/null +++ b/digital_asset_types/src/dao/generated/batch_mint.rs @@ -0,0 +1,58 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "batch_mint" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Serialize, Deserialize)] +pub struct Model { + pub file_hash: String, + pub batch_mint_binary_bincode: Vec, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + FileHash, + BatchMintBinaryBincode, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + FileHash, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = String; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::FileHash => ColumnType::String(None).def(), + Self::BatchMintBinaryBincode => ColumnType::Binary.def(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/digital_asset_types/src/dao/generated/batch_mint_to_verify.rs b/digital_asset_types/src/dao/generated/batch_mint_to_verify.rs new file mode 100644 index 000000000..d563a3107 --- /dev/null +++ b/digital_asset_types/src/dao/generated/batch_mint_to_verify.rs @@ -0,0 +1,84 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3 + +use super::sea_orm_active_enums::BatchMintFailStatus; +use super::sea_orm_active_enums::BatchMintPersistingState; +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Copy, Clone, Default, Debug, DeriveEntity)] +pub struct Entity; + +impl EntityName for Entity { + fn table_name(&self) -> &str { + "batch_mint_to_verify" + } +} + +#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Serialize, Deserialize)] +pub struct Model { + pub file_hash: String, + pub url: String, + pub created_at_slot: i64, + pub signature: String, + pub merkle_tree: Vec, + pub staker: Vec, + pub collection: Option>, + pub download_attempts: i32, + pub batch_mint_persisting_state: BatchMintPersistingState, + pub batch_mint_fail_status: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] +pub enum Column { + FileHash, + Url, + CreatedAtSlot, + Signature, + MerkleTree, + Staker, + Collection, + DownloadAttempts, + BatchMintPersistingState, + BatchMintFailStatus, +} + +#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)] +pub enum PrimaryKey { + FileHash, +} + +impl PrimaryKeyTrait for PrimaryKey { + type ValueType = String; + fn auto_increment() -> bool { + false + } +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl ColumnTrait for Column { + type EntityName = Entity; + fn def(&self) -> ColumnDef { + match self { + Self::FileHash => ColumnType::String(None).def(), + Self::Url => ColumnType::String(None).def(), + Self::CreatedAtSlot => ColumnType::BigInteger.def(), + Self::Signature => ColumnType::String(None).def(), + Self::MerkleTree => ColumnType::Binary.def(), + Self::Staker => ColumnType::Binary.def(), + Self::Collection => ColumnType::Binary.def().null(), + Self::DownloadAttempts => ColumnType::Integer.def(), + Self::BatchMintPersistingState => BatchMintPersistingState::db_type(), + Self::BatchMintFailStatus => BatchMintFailStatus::db_type().null(), + } + } +} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/digital_asset_types/src/dao/generated/mod.rs b/digital_asset_types/src/dao/generated/mod.rs index cb7314768..42dcb52e1 100644 --- a/digital_asset_types/src/dao/generated/mod.rs +++ b/digital_asset_types/src/dao/generated/mod.rs @@ -9,6 +9,8 @@ pub mod asset_data; pub mod asset_grouping; pub mod asset_v1_account_attachments; pub mod backfill_items; +pub mod batch_mint; +pub mod batch_mint_to_verify; pub mod cl_audits_v2; pub mod cl_items; pub mod raw_txn; diff --git a/digital_asset_types/src/dao/generated/prelude.rs b/digital_asset_types/src/dao/generated/prelude.rs index 4eabcb4a7..37cc418af 100644 --- a/digital_asset_types/src/dao/generated/prelude.rs +++ b/digital_asset_types/src/dao/generated/prelude.rs @@ -1,7 +1,5 @@ //! SeaORM Entity. Generated by sea-orm-codegen 0.9.3 -#![allow(unused_imports)] - pub use super::asset::Entity as Asset; pub use super::asset_authority::Entity as AssetAuthority; pub use super::asset_creators::Entity as AssetCreators; @@ -9,6 +7,8 @@ pub use super::asset_data::Entity as AssetData; pub use super::asset_grouping::Entity as AssetGrouping; pub use super::asset_v1_account_attachments::Entity as AssetV1AccountAttachments; pub use super::backfill_items::Entity as BackfillItems; +pub use super::batch_mint::Entity as BatchMint; +pub use super::batch_mint_to_verify::Entity as BatchMintToVerify; pub use super::cl_audits_v2::Entity as ClAuditsV2; pub use super::cl_items::Entity as ClItems; pub use super::raw_txn::Entity as RawTxn; diff --git a/digital_asset_types/src/dao/generated/sea_orm_active_enums.rs b/digital_asset_types/src/dao/generated/sea_orm_active_enums.rs index e4d0e012d..737607aee 100644 --- a/digital_asset_types/src/dao/generated/sea_orm_active_enums.rs +++ b/digital_asset_types/src/dao/generated/sea_orm_active_enums.rs @@ -20,38 +20,72 @@ pub enum SpecificationVersions { V2, } #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")] -pub enum TaskStatus { - #[sea_orm(string_value = "failed")] - Failed, - #[sea_orm(string_value = "pending")] - Pending, - #[sea_orm(string_value = "running")] - Running, - #[sea_orm(string_value = "success")] - Success, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "chain_mutability")] -pub enum ChainMutability { - #[sea_orm(string_value = "immutable")] - Immutable, - #[sea_orm(string_value = "mutable")] - Mutable, +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "instruction")] +pub enum Instruction { + #[sea_orm(string_value = "burn")] + Burn, + #[sea_orm(string_value = "cancel_redeem")] + CancelRedeem, + #[sea_orm(string_value = "compress")] + Compress, + #[sea_orm(string_value = "decompress_v1")] + DecompressV1, + #[sea_orm(string_value = "delegate")] + Delegate, + #[sea_orm(string_value = "mint_to_collection_v1")] + MintToCollectionV1, + #[sea_orm(string_value = "mint_v1")] + MintV1, + #[sea_orm(string_value = "redeem")] + Redeem, + #[sea_orm(string_value = "set_and_verify_collection")] + SetAndVerifyCollection, + #[sea_orm(string_value = "transfer")] + Transfer, #[sea_orm(string_value = "unknown")] Unknown, + #[sea_orm(string_value = "unverify_collection")] + UnverifyCollection, + #[sea_orm(string_value = "unverify_creator")] + UnverifyCreator, + #[sea_orm(string_value = "update_metadata")] + UpdateMetadata, + #[sea_orm(string_value = "verify_collection")] + VerifyCollection, + #[sea_orm(string_value = "verify_creator")] + VerifyCreator, } #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")] -pub enum Mutability { - #[sea_orm(string_value = "immutable")] - Immutable, - #[sea_orm(string_value = "mutable")] - Mutable, +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "owner_type")] +pub enum OwnerType { + #[sea_orm(string_value = "single")] + Single, + #[sea_orm(string_value = "token")] + Token, #[sea_orm(string_value = "unknown")] Unknown, } #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "batch_mint_persisting_state" +)] +pub enum BatchMintPersistingState { + #[sea_orm(string_value = "failed_to_persist")] + FailedToPersist, + #[sea_orm(string_value = "received_transaction")] + ReceivedTransaction, + #[sea_orm(string_value = "start_processing")] + StartProcessing, + #[sea_orm(string_value = "stored_update")] + StoredUpdate, + #[sea_orm(string_value = "successfully_download")] + SuccessfullyDownload, + #[sea_orm(string_value = "successfully_validate")] + SuccessfullyValidate, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] #[sea_orm( rs_type = "String", db_type = "Enum", @@ -70,32 +104,6 @@ pub enum V1AccountAttachments { Unknown, } #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "owner_type")] -pub enum OwnerType { - #[sea_orm(string_value = "single")] - Single, - #[sea_orm(string_value = "token")] - Token, - #[sea_orm(string_value = "unknown")] - Unknown, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm( - rs_type = "String", - db_type = "Enum", - enum_name = "royalty_target_type" -)] -pub enum RoyaltyTargetType { - #[sea_orm(string_value = "creators")] - Creators, - #[sea_orm(string_value = "fanout")] - Fanout, - #[sea_orm(string_value = "single")] - Single, - #[sea_orm(string_value = "unknown")] - Unknown, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] #[sea_orm( rs_type = "String", db_type = "Enum", @@ -128,38 +136,66 @@ pub enum SpecificationAssetClass { Unknown, } #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "instruction")] -pub enum Instruction { - #[sea_orm(string_value = "burn")] - Burn, - #[sea_orm(string_value = "cancel_redeem")] - CancelRedeem, - #[sea_orm(string_value = "compress")] - Compress, - #[sea_orm(string_value = "decompress_v1")] - DecompressV1, - #[sea_orm(string_value = "delegate")] - Delegate, - #[sea_orm(string_value = "mint_to_collection_v1")] - MintToCollectionV1, - #[sea_orm(string_value = "mint_v1")] - MintV1, - #[sea_orm(string_value = "redeem")] - Redeem, - #[sea_orm(string_value = "set_and_verify_collection")] - SetAndVerifyCollection, - #[sea_orm(string_value = "transfer")] - Transfer, +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "chain_mutability")] +pub enum ChainMutability { + #[sea_orm(string_value = "immutable")] + Immutable, + #[sea_orm(string_value = "mutable")] + Mutable, #[sea_orm(string_value = "unknown")] Unknown, - #[sea_orm(string_value = "unverify_collection")] - UnverifyCollection, - #[sea_orm(string_value = "unverify_creator")] - UnverifyCreator, - #[sea_orm(string_value = "update_metadata")] - UpdateMetadata, - #[sea_orm(string_value = "verify_collection")] - VerifyCollection, - #[sea_orm(string_value = "verify_creator")] - VerifyCreator, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")] +pub enum TaskStatus { + #[sea_orm(string_value = "failed")] + Failed, + #[sea_orm(string_value = "pending")] + Pending, + #[sea_orm(string_value = "running")] + Running, + #[sea_orm(string_value = "success")] + Success, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "royalty_target_type" +)] +pub enum RoyaltyTargetType { + #[sea_orm(string_value = "creators")] + Creators, + #[sea_orm(string_value = "fanout")] + Fanout, + #[sea_orm(string_value = "single")] + Single, + #[sea_orm(string_value = "unknown")] + Unknown, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")] +pub enum Mutability { + #[sea_orm(string_value = "immutable")] + Immutable, + #[sea_orm(string_value = "mutable")] + Mutable, + #[sea_orm(string_value = "unknown")] + Unknown, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "batch_mint_fail_status" +)] +pub enum BatchMintFailStatus { + #[sea_orm(string_value = "batch_mint_verify_failed")] + BatchMintVerifyFailed, + #[sea_orm(string_value = "checksum_verify_failed")] + ChecksumVerifyFailed, + #[sea_orm(string_value = "download_failed")] + DownloadFailed, + #[sea_orm(string_value = "file_serialization")] + FileSerialization, } diff --git a/digital_asset_types/tests/json_parsing.rs b/digital_asset_types/tests/json_parsing.rs index e630c91ff..ba0a3126a 100644 --- a/digital_asset_types/tests/json_parsing.rs +++ b/digital_asset_types/tests/json_parsing.rs @@ -10,7 +10,7 @@ use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; pub async fn load_test_json(file_name: &str) -> serde_json::Value { - let json = tokio::fs::read_to_string(format!("tools/data/{}", file_name)) + let json = tokio::fs::read_to_string(format!("tests/data/{}", file_name)) .await .unwrap(); serde_json::from_str(&json).unwrap() diff --git a/docker-compose.yaml b/docker-compose.yaml index 364a00855..2294fc4a4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -114,4 +114,4 @@ services: - "9900:9900" volumes: grafana_data: { } - graphite_data: { } + graphite_data: { } \ No newline at end of file diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 10c817a3e..cbbc9c253 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -33,3 +33,11 @@ spl-token = { workspace = true, features = ["no-entrypoint"] } sqlx = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } +mpl-bubblegum = { workspace = true } +spl-concurrent-merkle-tree = { workspace = true } +tempfile = { workspace = true } +async-channel = { workspace = true } +cadence = { workspace = true } +cadence-macros = { workspace = true } +bubblegum-batch-sdk = { workspace = true } +spl-account-compression = { workspace = true } \ No newline at end of file diff --git a/integration_tests/tests/integration_tests/batch_mint_tests.rs b/integration_tests/tests/integration_tests/batch_mint_tests.rs new file mode 100644 index 000000000..0ef24f33e --- /dev/null +++ b/integration_tests/tests/integration_tests/batch_mint_tests.rs @@ -0,0 +1,1381 @@ +use crate::common::Network; +use crate::common::TestSetup; +use crate::common::TestSetupOptions; +use borsh::BorshSerialize; +use bubblegum_batch_sdk::batch_mint_client::BatchMintClient; +use bubblegum_batch_sdk::batch_mint_validations::generate_batch_mint; +use bubblegum_batch_sdk::model::BatchMint; +use bubblegum_batch_sdk::model::CollectionConfig; +use cadence::{NopMetricSink, StatsdClient}; +use cadence_macros::set_global_default; +use das_api::api::ApiContract; +use das_api::api::GetAssetProof; +use digital_asset_types::dao::asset; +use digital_asset_types::dao::sea_orm_active_enums::{ + BatchMintFailStatus, BatchMintPersistingState, +}; +use digital_asset_types::dao::{batch_mint, batch_mint_to_verify}; +use flatbuffers::FlatBufferBuilder; +use mpl_bubblegum::types::Collection; +use mpl_bubblegum::types::Creator; +use mpl_bubblegum::types::TokenProgramVersion; +use mpl_bubblegum::types::Version; +use mpl_bubblegum::types::{LeafSchema, MetadataArgs}; +use mpl_bubblegum::utils::get_asset_id; +use mpl_bubblegum::LeafSchemaEvent; +use nft_ingester::batch_mint_updates::create_batch_mint_notification_channel; +use nft_ingester::plerkle::PlerkleTransactionInfo; +use plerkle_serialization::root_as_transaction_info; +use plerkle_serialization::serializer::serialize_transaction; +use program_transformers::batch_minting::batch_mint_persister::{ + BatchMintPersister, MockBatchMintDownloader, +}; +use program_transformers::error::BatchMintValidationError; +use program_transformers::ProgramTransformer; +use sea_orm::sea_query::OnConflict; +use sea_orm::{ColumnTrait, ConnectionTrait, DbBackend, IntoActiveModel, QueryTrait, Set}; +use sea_orm::{EntityTrait, QueryFilter}; +use solana_client::nonblocking::rpc_client::RpcClient; +use solana_sdk::instruction::CompiledInstruction; +use solana_sdk::keccak; +use solana_sdk::message::{Message, MessageHeader}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Keypair; +use solana_sdk::signature::Signature; +use solana_sdk::signer::Signer; +use solana_sdk::transaction::{SanitizedTransaction, Transaction}; +use solana_transaction_status::{InnerInstruction, InnerInstructions, TransactionStatusMeta}; +use spl_account_compression::events::ApplicationDataEvent; +use spl_account_compression::events::ApplicationDataEventV1; +use spl_account_compression::events::ChangeLogEventV1; +use spl_account_compression::state::PathNode; +use spl_account_compression::AccountCompressionEvent; +use spl_account_compression::ChangeLogEvent; +use spl_concurrent_merkle_tree::concurrent_merkle_tree::ConcurrentMerkleTree; +use std::collections::HashMap; +use std::fs::File; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; +use tokio::task::JoinSet; + +#[tokio::test] +async fn save_batch_mint_to_queue_test() { + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("save_batch_mint_to_queue_test".to_string()).await; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + + // arbitrary data + let batch_mint_instruction_data = + mpl_bubblegum::instructions::FinalizeTreeWithRootInstructionArgs { + root: [1; 32], + rightmost_leaf: [1; 32], + rightmost_index: 99, + metadata_url: metadata_url.clone(), + metadata_hash: metadata_hash.clone(), + }; + + // took it from Bubblegum client + // this value is generated by Anchor library, it's instruction identifier + let mut instruction_data = vec![77, 73, 220, 153, 126, 225, 64, 204]; + instruction_data.extend(batch_mint_instruction_data.try_to_vec().unwrap().iter()); + + let transaction = SanitizedTransaction::from_transaction_for_tests(Transaction { + signatures: vec![Signature::new_unique()], + message: Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![ + Pubkey::new_unique(), + Pubkey::from_str("BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY").unwrap(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ], + recent_blockhash: [1; 32].into(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0, 4, 2, 3, 5], + data: instruction_data, + }], + }, + }); + + // inner instruction is useless here but required by transaction parser + let transaction_status_meta = TransactionStatusMeta { + inner_instructions: Some(vec![InnerInstructions { + index: 0, + instructions: vec![InnerInstruction { + instruction: CompiledInstruction { + program_id_index: 2, + accounts: vec![], + data: vec![], + }, + stack_height: None, + }], + }]), + ..Default::default() + }; + + let transaction_info = + plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2 { + signature: &Signature::new_unique(), + is_vote: false, + transaction: &transaction, + transaction_status_meta: &transaction_status_meta, + index: 0, + }; + let builder = FlatBufferBuilder::new(); + let builder = serialize_transaction(builder, &transaction_info, 10); + let transaction_info = PlerkleTransactionInfo( + root_as_transaction_info(builder.finished_data().to_vec().as_slice()).unwrap(), + ) + .try_into() + .unwrap(); + + setup + .transformer + .handle_transaction(&transaction_info) + .await + .unwrap(); + + let r = batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap(); + + assert_eq!(r.file_hash, metadata_hash); + assert_eq!(r.url, metadata_url); +} + +#[tokio::test] +async fn skip_batched_minted_trees_test() { + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new_with_options( + "skip_batched_minted_trees_test".to_string(), + TestSetupOptions { + network: Some(Network::Devnet), + skip_batch_minted_trees: false, + }, + ) + .await; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + + // arbitrary data + let batch_mint_instruction_data = + mpl_bubblegum::instructions::FinalizeTreeWithRootInstructionArgs { + root: [1; 32], + rightmost_leaf: [1; 32], + rightmost_index: 99, + metadata_url: metadata_url.clone(), + metadata_hash: metadata_hash.clone(), + }; + + // took it from Bubblegum client + // this value is generated by Anchor library, it's instruction identifier + let mut instruction_data = vec![77, 73, 220, 153, 126, 225, 64, 204]; + instruction_data.extend(batch_mint_instruction_data.try_to_vec().unwrap().iter()); + + let merkle_tree_id = Pubkey::new_unique(); + + let transaction = SanitizedTransaction::from_transaction_for_tests(Transaction { + signatures: vec![Signature::new_unique()], + message: Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![ + Pubkey::new_unique(), + Pubkey::from_str("BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY").unwrap(), + Pubkey::new_unique(), + Pubkey::new_unique(), + merkle_tree_id, + Pubkey::new_unique(), + ], + recent_blockhash: [1; 32].into(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + // here important only 1th index - 4 + accounts: vec![0, 4, 2, 3, 5], + data: instruction_data, + }], + }, + }); + + // inner instruction is useless here but required by transaction parser + let transaction_status_meta = TransactionStatusMeta { + inner_instructions: Some(vec![InnerInstructions { + index: 0, + instructions: vec![InnerInstruction { + instruction: CompiledInstruction { + program_id_index: 2, + accounts: vec![], + data: vec![], + }, + stack_height: None, + }], + }]), + ..Default::default() + }; + + let transaction_info = + plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2 { + signature: &Signature::new_unique(), + is_vote: false, + transaction: &transaction, + transaction_status_meta: &transaction_status_meta, + index: 0, + }; + let builder = FlatBufferBuilder::new(); + let builder = serialize_transaction(builder, &transaction_info, 10); + let transaction_info = PlerkleTransactionInfo( + root_as_transaction_info(builder.finished_data().to_vec().as_slice()).unwrap(), + ) + .try_into() + .unwrap(); + + setup + .transformer + .handle_transaction(&transaction_info) + .await + .unwrap(); + + // mint asset and make sure we save it because program transformer is configured + // to save batched minted updates + mint_asset_to_tree(merkle_tree_id, &setup.transformer, 1, 1).await; + + let assets = asset::Entity::find().all(setup.db.as_ref()).await.unwrap(); + + assert_eq!(assets.len(), 1); + + // now we change program transformer and tell it not to save updates related to batched minted trees + // at this moment there is only 1 such tree + // also we make sure that during initialization program transformer selects all the batched tree ids from the DB + let setup = TestSetup::new_with_options( + "skip_batched_minted_trees_test".to_string(), + TestSetupOptions { + network: Some(Network::Devnet), + skip_batch_minted_trees: true, + }, + ) + .await; + + mint_asset_to_tree(merkle_tree_id, &setup.transformer, 2, 2).await; + + let assets = asset::Entity::find().all(setup.db.as_ref()).await.unwrap(); + + assert_eq!(assets.len(), 1); + + // now we switch back to saving batched mint tree updates and check if it works + let setup = TestSetup::new_with_options( + "skip_batched_minted_trees_test".to_string(), + TestSetupOptions { + network: Some(Network::Devnet), + skip_batch_minted_trees: false, + }, + ) + .await; + + mint_asset_to_tree(merkle_tree_id, &setup.transformer, 3, 3).await; + + let assets = asset::Entity::find().all(setup.db.as_ref()).await.unwrap(); + + assert_eq!(assets.len(), 2); +} + +async fn mint_asset_to_tree( + merkle_tree: Pubkey, + transformer: &ProgramTransformer, + index: u64, + sequence: u64, +) { + let mint_instruction_data = mpl_bubblegum::instructions::MintV1InstructionArgs { + metadata: MetadataArgs { + name: "name".to_string(), + symbol: "symbol".to_string(), + uri: "uri".to_string(), + seller_fee_basis_points: 1, + primary_sale_happened: false, + is_mutable: false, + edition_nonce: None, + token_standard: None, + collection: None, + uses: None, + token_program_version: TokenProgramVersion::Original, + creators: vec![], + }, + }; + + // took it from Bubblegum client + // this value is generated by Anchor library, it's instruction identifier + let mut instruction_data = vec![145, 98, 192, 118, 184, 147, 118, 104]; + instruction_data.extend(mint_instruction_data.try_to_vec().unwrap().iter()); + + let transaction = SanitizedTransaction::from_transaction_for_tests(Transaction { + signatures: vec![Signature::new_unique()], + message: Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![ + Pubkey::new_unique(), + Pubkey::from_str("BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY").unwrap(), + Pubkey::from_str("noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV").unwrap(), + Pubkey::new_unique(), + merkle_tree, + Pubkey::new_unique(), + ], + recent_blockhash: [1; 32].into(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + // here important only 3th index - 4 + accounts: vec![0, 3, 2, 4, 5], + data: instruction_data, + }], + }, + }); + + let change_log = ChangeLogEventV1 { + id: merkle_tree, + path: vec![ + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 32, + }, + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 16, + }, + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 8, + }, + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 4, + }, + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 2, + }, + PathNode { + node: Pubkey::new_unique().to_bytes(), + index: 1, + }, + ], + seq: sequence, + index: index as u32, + }; + + let change_log_event = AccountCompressionEvent::ChangeLog(ChangeLogEvent::V1(change_log)); + + let leaf_schema = LeafSchemaEvent::new( + Version::V1, + LeafSchema::V1 { + id: get_asset_id(&merkle_tree, index), + owner: Pubkey::new_unique(), + delegate: Pubkey::new_unique(), + nonce: index, + data_hash: Pubkey::new_unique().to_bytes(), + creator_hash: Pubkey::new_unique().to_bytes(), + }, + Pubkey::new_unique().to_bytes(), + ); + let leaf_schema_event = AccountCompressionEvent::ApplicationData(ApplicationDataEvent::V1( + ApplicationDataEventV1 { + application_data: leaf_schema.try_to_vec().unwrap(), + }, + )); + + // inner instruction is useless here but required by transaction parser + let transaction_status_meta = TransactionStatusMeta { + inner_instructions: Some(vec![InnerInstructions { + index: 0, + instructions: vec![ + InnerInstruction { + instruction: CompiledInstruction { + program_id_index: 2, + accounts: vec![], + data: change_log_event.try_to_vec().unwrap(), + }, + stack_height: None, + }, + InnerInstruction { + instruction: CompiledInstruction { + program_id_index: 2, + accounts: vec![], + data: leaf_schema_event.try_to_vec().unwrap(), + }, + stack_height: None, + }, + ], + }]), + ..Default::default() + }; + + let transaction_info = + plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2 { + signature: &Signature::new_unique(), + is_vote: false, + transaction: &transaction, + transaction_status_meta: &transaction_status_meta, + index: 0, + }; + let builder = FlatBufferBuilder::new(); + let builder = serialize_transaction(builder, &transaction_info, 10); + let transaction_info = PlerkleTransactionInfo( + root_as_transaction_info(builder.finished_data().to_vec().as_slice()).unwrap(), + ) + .try_into() + .unwrap(); + + transformer + .handle_transaction(&transaction_info) + .await + .unwrap(); +} + +fn generate_merkle_tree_from_batch_mint(batch_mint: &BatchMint) -> ConcurrentMerkleTree<10, 32> { + let mut merkle_tree = ConcurrentMerkleTree::<10, 32>::new(); + merkle_tree.initialize().unwrap(); + + for (nonce, asset) in batch_mint.batch_mints.iter().enumerate() { + let metadata_args_hash = keccak::hashv(&[asset.mint_args.try_to_vec().unwrap().as_slice()]); + let data_hash = keccak::hashv(&[ + &metadata_args_hash.to_bytes(), + &asset.mint_args.seller_fee_basis_points.to_le_bytes(), + ]); + + let creator_data = asset + .mint_args + .creators + .iter() + .map(|c| [c.address.as_ref(), &[c.verified as u8], &[c.share]].concat()) + .collect::>(); + + let creator_hash = keccak::hashv( + creator_data + .iter() + .map(|c| c.as_slice()) + .collect::>() + .as_ref(), + ); + + let id = mpl_bubblegum::utils::get_asset_id(&batch_mint.tree_id, nonce as u64); + + let leaf = LeafSchema::V1 { + id, + owner: Pubkey::from_str("3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM").unwrap(), + delegate: Pubkey::from_str("3VvLDXqJbw3heyRwFxv8MmurPznmDVUJS9gPMX2BDqfM").unwrap(), + nonce: nonce as u64, + data_hash: data_hash.to_bytes(), + creator_hash: creator_hash.to_bytes(), + }; + merkle_tree.append(leaf.hash()).unwrap(); + } + merkle_tree +} + +#[tokio::test] +async fn batch_mint_persister_test() { + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_persister_test".to_string()).await; + let test_batch_mint = generate_batch_mint(10); + let tmp_dir = tempfile::TempDir::new().unwrap(); + + let tmp_file = File::create(tmp_dir.path().join("batch-mint-10.json")).unwrap(); + serde_json::to_writer(tmp_file, &test_batch_mint).unwrap(); + + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(0), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(None), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint-10.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + let merkle_tree = generate_merkle_tree_from_batch_mint(&test_batch_mint); + + let leaf_index = 4u32; + + let payload = GetAssetProof { + id: test_batch_mint + .batch_mints + .get(leaf_index as usize) + .unwrap() + .leaf_update + .id() + .to_string(), + }; + let asset_proof = setup.das_api.get_asset_proof(payload).await.unwrap(); + let mut proofs: [[u8; 32]; 10] = [[0; 32]; 10]; + + for (i, s) in asset_proof.proof.iter().enumerate() { + proofs[i] = Pubkey::from_str(s).unwrap().to_bytes(); + } + + assert_eq!( + merkle_tree.check_valid_proof( + Pubkey::from_str(asset_proof.leaf.as_str()) + .unwrap() + .to_bytes(), + &proofs, + leaf_index + ), + true + ); + assert_eq!( + merkle_tree.check_valid_proof( + Pubkey::from_str(asset_proof.leaf.as_str()) + .unwrap() + .to_bytes(), + &proofs, + leaf_index + 1 + ), + false + ); + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::StoredUpdate + ); + + assert_eq!( + batch_mint::Entity::find() + .filter(batch_mint::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .is_some(), + true + ); +} + +#[tokio::test] +async fn batch_mint_persister_download_fail_test() { + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_persister_download_fail_test".to_string()).await; + let test_batch_mint = generate_batch_mint(10); + let tmp_dir = tempfile::TempDir::new().unwrap(); + let tmp_file = File::create(tmp_dir.path().join("batch-mint-10.json")).unwrap(); + serde_json::to_writer(tmp_file, &test_batch_mint).unwrap(); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(None), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + Err(BatchMintValidationError::Reqwest( + "Could not download file".to_string(), + )) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::FailedToPersist + ); + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_fail_status, + Some(BatchMintFailStatus::DownloadFailed) + ); +} + +#[tokio::test] +async fn batch_mint_with_verified_creators_test() { + // For this test it's necessary to use Solana mainnet RPC + let url = "https://api.mainnet-beta.solana.com".to_string(); + let solana_client = Arc::new(RpcClient::new_with_timeout(url, Duration::from_secs(3))); + // Merkle tree created in mainnet for testing purposes + let tree_key = Pubkey::from_str("AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb").unwrap(); + + // First we have to create offchain Merkle tree with SDK + + let batch_mint_client = BatchMintClient::new(solana_client); + let mut batch_mint_builder = batch_mint_client + .create_batch_mint_builder(&tree_key) + .await + .unwrap(); + + let asset_creator = Keypair::new(); + let owner = Keypair::new(); + let delegate = Keypair::new(); + + let asset = MetadataArgs { + name: "Name".to_string(), + symbol: "Symbol".to_string(), + uri: "https://immutable-storage/asset/".to_string(), + seller_fee_basis_points: 0, + primary_sale_happened: false, + is_mutable: false, + edition_nonce: None, + token_standard: Some(mpl_bubblegum::types::TokenStandard::NonFungible), + collection: None, + uses: None, + token_program_version: mpl_bubblegum::types::TokenProgramVersion::Original, + creators: vec![Creator { + address: asset_creator.pubkey(), + verified: true, + share: 100, + }], + }; + + let metadata_hash_arg = batch_mint_builder + .add_asset(&owner.pubkey(), &delegate.pubkey(), &asset) + .unwrap(); + + let signature = asset_creator.sign_message(&metadata_hash_arg.get_message()); + + let mut creators_signatures = HashMap::new(); + creators_signatures.insert(asset_creator.pubkey(), signature); + + let mut message_and_signatures = HashMap::new(); + message_and_signatures.insert(metadata_hash_arg.get_nonce(), creators_signatures); + + batch_mint_builder + .add_signatures_for_verified_creators(message_and_signatures) + .unwrap(); + + let finalized_batch_mint = batch_mint_builder.build_batch_mint().unwrap(); + + // Offchain Merkle tree creation is finished + // Start to process it + + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_with_verified_creators_test".to_string()).await; + + let tmp_dir = tempfile::TempDir::new().unwrap(); + let tmp_file = File::create(tmp_dir.path().join("batch-mint.json")).unwrap(); + serde_json::to_writer(tmp_file, &finalized_batch_mint).unwrap(); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(None), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::StoredUpdate + ); + + assert_eq!( + batch_mint::Entity::find() + .filter(batch_mint::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .is_some(), + true + ); +} + +#[tokio::test] +async fn batch_mint_with_unverified_creators_test() { + let setup = TestSetup::new("batch_mint_with_unverified_creators_test".to_string()).await; + // generate batch mint with creators verified value set to true + // but signatures will not be attached + // batch should not be saved + let mut test_batch_mint = generate_batch_mint(10); + + // set creators verified to true for this test case + for b_mint in test_batch_mint.batch_mints.iter_mut() { + for creator in b_mint.mint_args.creators.iter_mut() { + creator.verified = true; + } + } + + let tmp_dir = tempfile::TempDir::new().unwrap(); + + let tmp_file = File::create(tmp_dir.path().join("batch-mint-10.json")).unwrap(); + serde_json::to_writer(tmp_file, &test_batch_mint).unwrap(); + + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(None), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint-10.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::FailedToPersist + ); +} + +#[tokio::test] +async fn batch_mint_with_verified_collection_test() { + // For this test it's necessary to use Solana mainnet RPC + let url = "https://api.mainnet-beta.solana.com".to_string(); + let solana_client = Arc::new(RpcClient::new_with_timeout(url, Duration::from_secs(3))); + // Merkle tree created in mainnet for testing purposes + let tree_key = Pubkey::from_str("AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb").unwrap(); + + // First we have to create offchain Merkle tree with SDK + + let batch_mint_client = BatchMintClient::new(solana_client); + let mut batch_mint_builder = batch_mint_client + .create_batch_mint_builder(&tree_key) + .await + .unwrap(); + + let asset_creator = Keypair::new(); + let owner = Keypair::new(); + let delegate = Keypair::new(); + let collection_key = Pubkey::new_unique(); + + let collection_config = CollectionConfig { + collection_authority: Keypair::from_bytes(asset_creator.to_bytes().as_ref()).unwrap(), + collection_authority_record_pda: None, + collection_mint: collection_key, + collection_metadata: Pubkey::new_unique(), // doesn't matter in this case + edition_account: Pubkey::new_unique(), // doesn't matter in this case + }; + batch_mint_builder.setup_collection_config(collection_config); + + let asset = MetadataArgs { + name: "Name".to_string(), + symbol: "Symbol".to_string(), + uri: "https://immutable-storage/asset/".to_string(), + seller_fee_basis_points: 0, + primary_sale_happened: false, + is_mutable: false, + edition_nonce: None, + token_standard: Some(mpl_bubblegum::types::TokenStandard::NonFungible), + collection: Some(Collection { + verified: true, + key: collection_key, + }), + uses: None, + token_program_version: mpl_bubblegum::types::TokenProgramVersion::Original, + creators: vec![Creator { + address: asset_creator.pubkey(), + verified: false, + share: 100, + }], + }; + + let _ = batch_mint_builder + .add_asset(&owner.pubkey(), &delegate.pubkey(), &asset) + .unwrap(); + + let finalized_batch_mint = batch_mint_builder.build_batch_mint().unwrap(); + + // Offchain Merkle tree creation is finished + // Start to process it + + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_with_verified_collection_test".to_string()).await; + + let tmp_dir = tempfile::TempDir::new().unwrap(); + let tmp_file = File::create(tmp_dir.path().join("batch-mint.json")).unwrap(); + serde_json::to_writer(tmp_file, &finalized_batch_mint).unwrap(); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(Some(collection_key.to_bytes().to_vec())), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::StoredUpdate + ); + + assert_eq!( + batch_mint::Entity::find() + .filter(batch_mint::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .is_some(), + true + ); +} + +#[tokio::test] +async fn batch_mint_with_wrong_collection_test() { + // For this test it's necessary to use Solana mainnet RPC + let url = "https://api.mainnet-beta.solana.com".to_string(); + let solana_client = Arc::new(RpcClient::new_with_timeout(url, Duration::from_secs(3))); + // Merkle tree created in mainnet for testing purposes + let tree_key = Pubkey::from_str("AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb").unwrap(); + + // First we have to create offchain Merkle tree with SDK + + let batch_mint_client = BatchMintClient::new(solana_client); + let mut batch_mint_builder = batch_mint_client + .create_batch_mint_builder(&tree_key) + .await + .unwrap(); + + let asset_creator = Keypair::new(); + let owner = Keypair::new(); + let delegate = Keypair::new(); + let collection_key = Pubkey::new_unique(); + + let wrong_collection_key = Pubkey::new_unique(); + + let collection_config = CollectionConfig { + collection_authority: Keypair::from_bytes(asset_creator.to_bytes().as_ref()).unwrap(), + collection_authority_record_pda: None, + collection_mint: collection_key, + collection_metadata: Pubkey::new_unique(), // doesn't matter in this case + edition_account: Pubkey::new_unique(), // doesn't matter in this case + }; + batch_mint_builder.setup_collection_config(collection_config); + + let asset = MetadataArgs { + name: "Name".to_string(), + symbol: "Symbol".to_string(), + uri: "https://immutable-storage/asset/".to_string(), + seller_fee_basis_points: 0, + primary_sale_happened: false, + is_mutable: false, + edition_nonce: None, + token_standard: Some(mpl_bubblegum::types::TokenStandard::NonFungible), + collection: Some(Collection { + verified: true, + key: collection_key, + }), + uses: None, + token_program_version: mpl_bubblegum::types::TokenProgramVersion::Original, + creators: vec![Creator { + address: asset_creator.pubkey(), + verified: false, + share: 100, + }], + }; + + let _ = batch_mint_builder + .add_asset(&owner.pubkey(), &delegate.pubkey(), &asset) + .unwrap(); + + let finalized_batch_mint = batch_mint_builder.build_batch_mint().unwrap(); + + // Offchain Merkle tree creation is finished + // Start to process it + + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_with_verified_collection_test".to_string()).await; + + let tmp_dir = tempfile::TempDir::new().unwrap(); + let tmp_file = File::create(tmp_dir.path().join("batch-mint.json")).unwrap(); + serde_json::to_writer(tmp_file, &finalized_batch_mint).unwrap(); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(Some(wrong_collection_key.to_bytes().to_vec())), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::FailedToPersist + ); +} + +#[tokio::test] +async fn batch_mint_with_unverified_collection_test() { + // For this test it's necessary to use Solana mainnet RPC + let url = "https://api.mainnet-beta.solana.com".to_string(); + let solana_client = Arc::new(RpcClient::new_with_timeout(url, Duration::from_secs(3))); + // Merkle tree created in mainnet for testing purposes + let tree_key = Pubkey::from_str("AGMiLKtXX7PiVneM8S1KkTmCnF7X5zh6bKq4t1Mhrwpb").unwrap(); + + // First we have to create offchain Merkle tree with SDK + + let batch_mint_client = BatchMintClient::new(solana_client); + let mut batch_mint_builder = batch_mint_client + .create_batch_mint_builder(&tree_key) + .await + .unwrap(); + + let asset_creator = Keypair::new(); + let owner = Keypair::new(); + let delegate = Keypair::new(); + let collection_key = Pubkey::new_unique(); + + let collection_config = CollectionConfig { + collection_authority: Keypair::from_bytes(asset_creator.to_bytes().as_ref()).unwrap(), + collection_authority_record_pda: None, + collection_mint: collection_key, + collection_metadata: Pubkey::new_unique(), // doesn't matter in this case + edition_account: Pubkey::new_unique(), // doesn't matter in this case + }; + batch_mint_builder.setup_collection_config(collection_config); + + let asset = MetadataArgs { + name: "Name".to_string(), + symbol: "Symbol".to_string(), + uri: "https://immutable-storage/asset/".to_string(), + seller_fee_basis_points: 0, + primary_sale_happened: false, + is_mutable: false, + edition_nonce: None, + token_standard: Some(mpl_bubblegum::types::TokenStandard::NonFungible), + collection: Some(Collection { + verified: true, + key: collection_key, + }), + uses: None, + token_program_version: mpl_bubblegum::types::TokenProgramVersion::Original, + creators: vec![Creator { + address: asset_creator.pubkey(), + verified: false, + share: 100, + }], + }; + + let _ = batch_mint_builder + .add_asset(&owner.pubkey(), &delegate.pubkey(), &asset) + .unwrap(); + + let finalized_batch_mint = batch_mint_builder.build_batch_mint().unwrap(); + + // Offchain Merkle tree creation is finished + // Start to process it + + let client = StatsdClient::builder("batch_mint.test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + + set_global_default(client); + + let setup = TestSetup::new("batch_mint_with_verified_collection_test".to_string()).await; + + let tmp_dir = tempfile::TempDir::new().unwrap(); + let tmp_file = File::create(tmp_dir.path().join("batch-mint.json")).unwrap(); + serde_json::to_writer(tmp_file, &finalized_batch_mint).unwrap(); + + let download_attempts = 0; + let metadata_url = "url".to_string(); + let metadata_hash = "hash".to_string(); + let batch_mint_to_verify = batch_mint_to_verify::ActiveModel { + file_hash: Set(metadata_hash.clone()), + url: Set(metadata_url.clone()), + created_at_slot: Set(10), + signature: Set(Signature::new_unique().to_string()), + merkle_tree: Set(Pubkey::default().to_bytes().to_vec()), + staker: Set(Pubkey::default().to_bytes().to_vec()), + download_attempts: Set(download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(None), + } + .into_active_model(); + + let query = batch_mint_to_verify::Entity::insert(batch_mint_to_verify) + .on_conflict( + OnConflict::columns([batch_mint_to_verify::Column::FileHash]) + .update_columns([batch_mint_to_verify::Column::Url]) + .update_columns([batch_mint_to_verify::Column::Signature]) + .update_columns([batch_mint_to_verify::Column::DownloadAttempts]) + .update_columns([batch_mint_to_verify::Column::BatchMintFailStatus]) + .update_columns([batch_mint_to_verify::Column::BatchMintPersistingState]) + .update_columns([batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + setup.db.execute(query).await.unwrap(); + + let mut mocked_downloader = MockBatchMintDownloader::new(); + mocked_downloader + .expect_download_batch_mint_and_check_checksum() + .returning(move |_, _| { + let json_file = + std::fs::read_to_string(tmp_dir.path().join("batch-mint.json")).unwrap(); + Ok(Box::new(serde_json::from_str(&json_file).unwrap())) + }); + + let mut tasks = JoinSet::new(); + let r = create_batch_mint_notification_channel(&setup.database_test_url, &mut tasks) + .await + .unwrap(); + let batch_mint_persister = BatchMintPersister::new(setup.db.clone(), r, mocked_downloader); + let (batch_mint_to_verify, _) = batch_mint_persister + .get_batch_mint_to_verify() + .await + .unwrap(); + batch_mint_persister + .persist_batch_mint(batch_mint_to_verify.unwrap(), None) + .await; + + assert_eq!( + batch_mint_to_verify::Entity::find() + .filter(batch_mint_to_verify::Column::FileHash.eq(metadata_hash.clone())) + .one(setup.db.as_ref()) + .await + .unwrap() + .unwrap() + .batch_mint_persisting_state, + BatchMintPersistingState::FailedToPersist + ); +} diff --git a/integration_tests/tests/integration_tests/cnft_tests.rs b/integration_tests/tests/integration_tests/cnft_tests.rs index 5c772d30d..7868ed194 100644 --- a/integration_tests/tests/integration_tests/cnft_tests.rs +++ b/integration_tests/tests/integration_tests/cnft_tests.rs @@ -70,6 +70,7 @@ async fn test_cnft_scenario_mint_update_metadata() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -95,6 +96,7 @@ async fn test_cnft_scenario_mint_update_metadata_remove_creators() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -180,6 +182,7 @@ async fn test_mint_delegate_transfer() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -204,6 +207,7 @@ async fn test_mint_redeem_cancel_redeem() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -246,6 +250,7 @@ async fn test_mint_transfer_burn() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -288,6 +293,7 @@ async fn test_mint_transfer_transfer() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -312,6 +318,7 @@ async fn test_mint_verify_creator() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -335,6 +342,7 @@ async fn test_mint_verify_collection() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; diff --git a/integration_tests/tests/integration_tests/common.rs b/integration_tests/tests/integration_tests/common.rs index e190b39fa..19ce409ee 100644 --- a/integration_tests/tests/integration_tests/common.rs +++ b/integration_tests/tests/integration_tests/common.rs @@ -66,6 +66,7 @@ pub struct TestSetup { pub db: Arc, pub transformer: ProgramTransformer, pub das_api: DasApi, + pub database_test_url: String, } impl TestSetup { @@ -84,7 +85,8 @@ impl TestSetup { let pool = setup_pg_pool(database_test_url.clone()).await; let db = SqlxPostgresConnector::from_sqlx_postgres_pool(pool.clone()); - let transformer = load_ingest_program_transformer(pool.clone()).await; + let transformer = + load_ingest_program_transformer(pool.clone(), opts.skip_batch_minted_trees).await; let rpc_url = match opts.network.unwrap_or_default() { Network::Mainnet => std::env::var("MAINNET_RPC_URL").unwrap(), @@ -104,6 +106,7 @@ impl TestSetup { name, client, db: Arc::new(db), + database_test_url, transformer, das_api, } @@ -113,6 +116,7 @@ impl TestSetup { #[derive(Clone, Copy, Default)] pub struct TestSetupOptions { pub network: Option, + pub skip_batch_minted_trees: bool, } pub async fn setup_pg_pool(database_url: String) -> PgPool { @@ -161,8 +165,17 @@ pub async fn apply_migrations_and_delete_data(db: Arc) { .unwrap(); } -async fn load_ingest_program_transformer(pool: sqlx::Pool) -> ProgramTransformer { - ProgramTransformer::new(pool, Box::new(|_info| ready(Ok(())).boxed()), false) +async fn load_ingest_program_transformer( + pool: sqlx::Pool, + skip_batch_minted_trees: bool, +) -> ProgramTransformer { + ProgramTransformer::new( + pool, + Box::new(|_info| ready(Ok(())).boxed()), + false, + skip_batch_minted_trees, + ) + .await } pub async fn get_transaction( diff --git a/integration_tests/tests/integration_tests/main.rs b/integration_tests/tests/integration_tests/main.rs index 7220ca96c..7b73db395 100644 --- a/integration_tests/tests/integration_tests/main.rs +++ b/integration_tests/tests/integration_tests/main.rs @@ -1,4 +1,5 @@ mod account_update_tests; +mod batch_mint_tests; mod cnft_tests; mod common; mod general_scenario_tests; diff --git a/integration_tests/tests/integration_tests/mpl_core_tests.rs b/integration_tests/tests/integration_tests/mpl_core_tests.rs index a8630e84c..b6699018a 100644 --- a/integration_tests/tests/integration_tests/mpl_core_tests.rs +++ b/integration_tests/tests/integration_tests/mpl_core_tests.rs @@ -17,6 +17,7 @@ async fn test_mpl_core_get_asset() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -46,6 +47,7 @@ async fn test_mpl_core_get_collection() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -75,6 +77,7 @@ async fn test_mpl_core_get_assets_by_authority() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -117,6 +120,7 @@ async fn test_mpl_core_get_assets_by_group() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -158,6 +162,7 @@ async fn test_mpl_core_get_assets_by_owner() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -196,6 +201,7 @@ async fn test_mpl_core_get_asset_with_edition() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -225,6 +231,7 @@ async fn test_mpl_core_get_asset_with_pubkey_in_rule_set() { name.clone(), TestSetupOptions { network: Some(Network::Mainnet), + skip_batch_minted_trees: false, }, ) .await; @@ -254,6 +261,7 @@ async fn test_mpl_core_get_asset_with_two_oracle_external_plugins() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -283,6 +291,7 @@ async fn test_mpl_core_get_asset_with_oracle_external_plugin_on_collection() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -312,6 +321,7 @@ async fn test_mpl_core_get_asset_with_oracle_multiple_lifecycle_events() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -341,6 +351,7 @@ async fn test_mpl_core_get_asset_with_oracle_custom_offset_and_base_address_conf name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -370,6 +381,7 @@ async fn test_mpl_core_get_asset_with_oracle_no_offset() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -399,6 +411,7 @@ async fn test_mpl_core_get_assets_by_group_with_oracle_and_custom_pda_all_seeds( name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; @@ -438,6 +451,7 @@ async fn test_mpl_core_get_asset_with_multiple_internal_and_external_plugins() { name.clone(), TestSetupOptions { network: Some(Network::Devnet), + skip_batch_minted_trees: false, }, ) .await; diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 9b4689126..5082e65da 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -42,6 +42,7 @@ mod m20240313_120101_add_mpl_core_plugins_columns; mod m20240319_120101_add_mpl_core_enum_vals; mod m20240320_120101_add_mpl_core_info_items; mod m20240520_120101_add_mpl_core_external_plugins_columns; +mod m20240720_120101_add_finalize_tree_with_root_instruction_handle; pub mod model; @@ -93,6 +94,7 @@ impl MigratorTrait for Migrator { Box::new(m20240319_120101_add_mpl_core_enum_vals::Migration), Box::new(m20240320_120101_add_mpl_core_info_items::Migration), Box::new(m20240520_120101_add_mpl_core_external_plugins_columns::Migration), + Box::new(m20240720_120101_add_finalize_tree_with_root_instruction_handle::Migration), ] } } diff --git a/migration/src/m20240720_120101_add_finalize_tree_with_root_instruction_handle.rs b/migration/src/m20240720_120101_add_finalize_tree_with_root_instruction_handle.rs new file mode 100644 index 000000000..f45bb86c8 --- /dev/null +++ b/migration/src/m20240720_120101_add_finalize_tree_with_root_instruction_handle.rs @@ -0,0 +1,232 @@ +use crate::sea_orm::{DatabaseBackend, Statement}; +use enum_iterator::all; +use enum_iterator_derive::Sequence; +use sea_orm::sea_query::extension::postgres::Type; +use sea_orm_migration::prelude::*; +use sea_orm_migration::sea_orm::ConnectionTrait; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_type( + Type::create() + .as_enum(BatchMintToVerify::BatchMintFailStatus) + .values([ + FailedBatchMintState::ChecksumVerifyFailed, + FailedBatchMintState::BatchMintVerifyFailed, + FailedBatchMintState::DownloadFailed, + FailedBatchMintState::FileSerialization, + ]) + .to_owned(), + ) + .await?; + + manager + .create_type( + Type::create() + .as_enum(BatchMintToVerify::BatchMintPersistingState) + .values([ + PersistingBatchMintState::ReceivedTransaction, + PersistingBatchMintState::StartProcessing, + PersistingBatchMintState::FailedToPersist, + PersistingBatchMintState::SuccessfullyDownload, + PersistingBatchMintState::SuccessfullyValidate, + PersistingBatchMintState::StoredUpdate, + ]) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(BatchMintToVerify::Table) + .if_not_exists() + .col( + ColumnDef::new(BatchMintToVerify::FileHash) + .string() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(BatchMintToVerify::Url).string().not_null()) + .col( + ColumnDef::new(BatchMintToVerify::CreatedAtSlot) + .big_integer() + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::Signature) + .string() + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::MerkleTree) + .binary() + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::Staker) + .binary() + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::Collection) + .binary() + .null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::DownloadAttempts) + .unsigned() + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::BatchMintPersistingState) + .enumeration( + BatchMintToVerify::BatchMintPersistingState, + all::().collect::>(), + ) + .not_null(), + ) + .col( + ColumnDef::new(BatchMintToVerify::BatchMintFailStatus) + .enumeration( + BatchMintToVerify::BatchMintFailStatus, + all::().collect::>(), + ) + .null(), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("idx_created_at_slot") + .table(BatchMintToVerify::Table) + .col(BatchMintToVerify::CreatedAtSlot) + .col(BatchMintToVerify::BatchMintPersistingState) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("idx_batch_mint_tree") + .table(BatchMintToVerify::Table) + .col(BatchMintToVerify::MerkleTree) + .to_owned(), + ) + .await?; + + manager + .get_connection() + .execute(Statement::from_string( + DatabaseBackend::Postgres, + "ALTER TABLE batch_mint_to_verify ADD CONSTRAINT unique_filehash_staker UNIQUE (file_hash, staker);".to_string(), + )) + .await?; + + manager + .create_table( + Table::create() + .table(BatchMint::Table) + .if_not_exists() + .col( + ColumnDef::new(BatchMint::FileHash) + .string() + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(BatchMint::BatchMintBinaryBincode) + .binary() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .get_connection() + .execute(Statement::from_string( + DatabaseBackend::Postgres, + "CREATE FUNCTION notify_new_batch_mint() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM pg_notify('new_batch_mint', NEW::text); + RETURN NEW; + END; + $$; + CREATE TRIGGER batch_mint_to_verify_trigger + AFTER INSERT ON batch_mint_to_verify + FOR EACH ROW EXECUTE FUNCTION notify_new_batch_mint();" + .to_string(), + )) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute(Statement::from_string( + DatabaseBackend::Postgres, + "DROP TRIGGER IF EXISTS batch_mint_to_verify_trigger ON batch_mint_to_verify; + DROP FUNCTION IF EXISTS notify_new_batch_mint;" + .to_string(), + )) + .await?; + manager + .drop_table(Table::drop().table(BatchMintToVerify::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(BatchMint::Table).to_owned()) + .await?; + Ok(()) + } +} + +#[derive(Iden)] +enum BatchMintToVerify { + Table, + Url, + FileHash, + CreatedAtSlot, + Signature, + MerkleTree, + DownloadAttempts, + BatchMintPersistingState, + BatchMintFailStatus, + Staker, + Collection, +} + +#[derive(Iden, Debug, PartialEq, Sequence)] +enum PersistingBatchMintState { + ReceivedTransaction, + FailedToPersist, + StartProcessing, + SuccessfullyDownload, + SuccessfullyValidate, + StoredUpdate, +} + +#[derive(Iden, Debug, PartialEq, Sequence)] +enum FailedBatchMintState { + DownloadFailed, + ChecksumVerifyFailed, + BatchMintVerifyFailed, + FileSerialization, +} + +#[derive(Iden)] +enum BatchMint { + Table, + FileHash, + BatchMintBinaryBincode, +} diff --git a/nft_ingester/Cargo.toml b/nft_ingester/Cargo.toml index fd17a2bed..2b8709a77 100644 --- a/nft_ingester/Cargo.toml +++ b/nft_ingester/Cargo.toml @@ -60,6 +60,8 @@ tracing-subscriber = { workspace = true, features = [ "ansi", ] } url = { workspace = true } +triomphe = { workspace = true } +async-channel = { workspace = true } [lints] workspace = true diff --git a/nft_ingester/src/account_updates.rs b/nft_ingester/src/account_updates.rs index 6d5372555..1a53bfc58 100644 --- a/nft_ingester/src/account_updates.rs +++ b/nft_ingester/src/account_updates.rs @@ -31,11 +31,17 @@ pub fn account_worker( tokio::spawn(async move { let source = T::new(config).await; if let Ok(mut msg) = source { - let manager = Arc::new(ProgramTransformer::new( - pool, - create_download_metadata_notifier(bg_task_sender), - false, - )); + let manager = Arc::new( + // here we hardcode cl_audits and skip_batch_minted_trees to false because that flags are not + // used during accounts processing and that values can be arbitrary + ProgramTransformer::new( + pool, + create_download_metadata_notifier(bg_task_sender), + false, + false, + ) + .await, + ); loop { let e = msg.recv(stream_key, consumption_type.clone()).await; let mut tasks = JoinSet::new(); diff --git a/nft_ingester/src/batch_mint_updates.rs b/nft_ingester/src/batch_mint_updates.rs new file mode 100644 index 000000000..3f773bf6e --- /dev/null +++ b/nft_ingester/src/batch_mint_updates.rs @@ -0,0 +1,29 @@ +use crate::error::IngesterError; +use async_channel::Receiver; +use log::error; +use sqlx::postgres::PgListener; +use tokio::task::{JoinError, JoinSet}; + +const BATCH_MINT_LISTEN_KEY: &str = "new_batch_mint"; + +pub async fn create_batch_mint_notification_channel( + database_url: &str, + tasks: &mut JoinSet>, +) -> Result, IngesterError> { + let mut listener = PgListener::connect(database_url).await?; + listener.listen(BATCH_MINT_LISTEN_KEY).await?; + let (s, r) = async_channel::unbounded::<()>(); + tasks.spawn(async move { + loop { + if let Err(e) = listener.recv().await { + error!("Recv batch mint notification: {}", e); + continue; + } + if let Err(e) = s.send(()).await { + error!("Send batch mint notification: {}", e); + } + } + }); + + Ok(r) +} diff --git a/nft_ingester/src/config.rs b/nft_ingester/src/config.rs index ab1f3cd38..b7e8c6d62 100644 --- a/nft_ingester/src/config.rs +++ b/nft_ingester/src/config.rs @@ -31,6 +31,7 @@ pub struct IngesterConfig { pub code_version: Option<&'static str>, pub background_task_runner_config: Option, pub cl_audits: Option, // save transaction logs for compressed nfts + pub skip_batch_mint_indexing: bool, } #[derive(Deserialize, PartialEq, Debug, Clone)] @@ -44,6 +45,7 @@ pub struct WorkerConfig { pub enum WorkerType { Account, Transaction, + Rollup, } impl IngesterConfig { @@ -91,6 +93,11 @@ impl IngesterConfig { worker_count: 2, worker_type: WorkerType::Transaction, }, + WorkerConfig { + stream_name: "RLP".to_string(), + worker_count: 1, + worker_type: WorkerType::Rollup, + }, ] } } diff --git a/nft_ingester/src/error/mod.rs b/nft_ingester/src/error/mod.rs index 37ed5f24b..9a7d60151 100644 --- a/nft_ingester/src/error/mod.rs +++ b/nft_ingester/src/error/mod.rs @@ -52,6 +52,8 @@ pub enum IngesterError { HttpError { status_code: String }, #[error("AssetIndex Error {0}")] AssetIndexError(String), + #[error("sqlx Error {0}")] + SqlxError(String), } impl From for IngesterError { @@ -113,3 +115,9 @@ impl From for IngesterError { IngesterError::SerializatonError(e.to_string()) } } + +impl From for IngesterError { + fn from(e: sqlx::Error) -> Self { + IngesterError::SqlxError(e.to_string()) + } +} diff --git a/nft_ingester/src/lib.rs b/nft_ingester/src/lib.rs index 9569da43a..8c2a44e13 100644 --- a/nft_ingester/src/lib.rs +++ b/nft_ingester/src/lib.rs @@ -1,5 +1,6 @@ pub mod ack; pub mod backfiller; +pub mod batch_mint_updates; pub mod config; pub mod database; pub mod error; diff --git a/nft_ingester/src/main.rs b/nft_ingester/src/main.rs index caf2c26f1..3cf830d2e 100644 --- a/nft_ingester/src/main.rs +++ b/nft_ingester/src/main.rs @@ -26,7 +26,13 @@ use cadence_macros::{is_global_default_set, statsd_count}; use chrono::Duration; use clap::{arg, command, value_parser}; use log::{error, info}; +use nft_ingester::batch_mint_updates; use plerkle_messenger::{redis_messenger::RedisMessenger, ConsumptionType}; +use program_transformers::batch_minting::batch_mint_persister::{ + BatchMintDownloaderForPersister, BatchMintPersister, +}; +use sea_orm::{DatabaseConnection, SqlxPostgresConnector}; +use std::sync::Arc; use std::{path::PathBuf, time}; use tokio::{signal, task::JoinSet}; @@ -95,6 +101,24 @@ pub async fn main() -> Result<(), IngesterError> { if role != IngesterRole::BackgroundTaskRunner { tasks.spawn(bg_task_listener); } + let mut rollup_persister: Option< + Arc>, + > = None; + if !config.skip_batch_mint_indexing { + let r = batch_mint_updates::create_batch_mint_notification_channel( + &config.get_database_url(), + &mut tasks, + ) + .await + .unwrap(); + rollup_persister = Some(Arc::new(BatchMintPersister::new( + Arc::new(SqlxPostgresConnector::from_sqlx_postgres_pool( + database_pool.clone(), + )), + r, + BatchMintDownloaderForPersister {}, + ))); + } // Stream Consumers Setup ------------------------------------- if role == IngesterRole::Ingester || role == IngesterRole::All { @@ -118,33 +142,45 @@ pub async fn main() -> Result<(), IngesterError> { } for i in 0..worker.worker_count { - if worker.worker_type == WorkerType::Account { - let _account = account_worker::( - database_pool.clone(), - config.get_messneger_client_config(), - bg_task_sender.clone(), - ack_sender.clone(), - if i == 0 { - ConsumptionType::Redeliver - } else { - ConsumptionType::New - }, - stream_name, - ); - } else if worker.worker_type == WorkerType::Transaction { - let _txn = transaction_worker::( - database_pool.clone(), - config.get_messneger_client_config(), - bg_task_sender.clone(), - ack_sender.clone(), - if i == 0 { - ConsumptionType::Redeliver - } else { - ConsumptionType::New - }, - config.cl_audits.unwrap_or(false), - stream_name, - ); + match worker.worker_type { + WorkerType::Account => { + let _account = account_worker::( + database_pool.clone(), + config.get_messneger_client_config(), + bg_task_sender.clone(), + ack_sender.clone(), + if i == 0 { + ConsumptionType::Redeliver + } else { + ConsumptionType::New + }, + stream_name, + ); + } + WorkerType::Transaction => { + let _txn = transaction_worker::( + database_pool.clone(), + config.get_messneger_client_config(), + bg_task_sender.clone(), + ack_sender.clone(), + if i == 0 { + ConsumptionType::Redeliver + } else { + ConsumptionType::New + }, + config.cl_audits.unwrap_or(false), + stream_name, + config.skip_batch_mint_indexing, + ); + } + WorkerType::Rollup => { + if let Some(rollup_persister) = rollup_persister.clone() { + tasks.spawn(async move { + rollup_persister.persist_batch_mints().await; + Ok(()) + }); + } + } } } } @@ -174,6 +210,7 @@ pub async fn main() -> Result<(), IngesterError> { } } + // Don`t we need to use some graceful stop mechanism such as cancellation token? tasks.shutdown().await; Ok(()) diff --git a/nft_ingester/src/transaction_notifications.rs b/nft_ingester/src/transaction_notifications.rs index df005bc56..3351ee6fb 100644 --- a/nft_ingester/src/transaction_notifications.rs +++ b/nft_ingester/src/transaction_notifications.rs @@ -28,15 +28,20 @@ pub fn transaction_worker( consumption_type: ConsumptionType, cl_audits: bool, stream_key: &'static str, + skip_batch_minted_trees: bool, ) -> JoinHandle<()> { tokio::spawn(async move { let source = T::new(config).await; if let Ok(mut msg) = source { - let manager = Arc::new(ProgramTransformer::new( - pool, - create_download_metadata_notifier(bg_task_sender), - cl_audits, - )); + let manager = Arc::new( + ProgramTransformer::new( + pool, + create_download_metadata_notifier(bg_task_sender), + cl_audits, + skip_batch_minted_trees, + ) + .await, + ); loop { let e = msg.recv(stream_key, consumption_type.clone()).await; let mut tasks = JoinSet::new(); diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index 35bab7a19..86b25f22d 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -25,6 +25,23 @@ sqlx = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["time"] } tracing = { workspace = true } +triomphe = { workspace = true } +spl-concurrent-merkle-tree = { workspace = true } +paste = { workspace = true } +async-trait = { workspace = true } +serde_derive = { workspace = true } +anchor-lang = { workspace = true } +mockall = { workspace = true } +reqwest = { workspace = true } +hex = { workspace = true } +xxhash-rust = { workspace = true } +serde = { workspace = true } +serde_with = { workspace = true } +bincode = { workspace = true } +rand = { workspace = true } +async-channel = { workspace = true } +bubblegum-batch-sdk = { workspace = true } +solana-client = { workspace = true } [lints] workspace = true diff --git a/program_transformers/src/batch_minting/batch_mint_persister.rs b/program_transformers/src/batch_minting/batch_mint_persister.rs new file mode 100644 index 000000000..eb3ad6655 --- /dev/null +++ b/program_transformers/src/batch_minting/batch_mint_persister.rs @@ -0,0 +1,484 @@ +use async_channel::Receiver; +use std::sync::Arc; + +use crate::bubblegum; +use crate::error::{BatchMintValidationError, ProgramTransformerError, ProgramTransformerResult}; +use async_trait::async_trait; +use blockbuster::instruction::InstructionBundle; +use bubblegum_batch_sdk::{batch_mint_validations::validate_batch_mint, model::BatchMint}; +use cadence_macros::{statsd_count, statsd_histogram}; +use digital_asset_types::dao::sea_orm_active_enums::{ + BatchMintFailStatus, BatchMintPersistingState, +}; +use digital_asset_types::dao::{batch_mint, batch_mint_to_verify}; +use mockall::automock; +use sea_orm::sea_query::{LockType, OnConflict}; +use sea_orm::ActiveValue::Set; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, DbBackend, EntityTrait, + IntoActiveModel, QueryFilter, QueryOrder, QuerySelect, QueryTrait, TransactionTrait, +}; +use solana_sdk::pubkey::Pubkey; +use tokio::time::Instant; +use tracing::{error, info}; + +pub const MAX_BATCH_MINT_DOWNLOAD_ATTEMPTS: u8 = 5; + +#[automock] +#[async_trait] +pub trait BatchMintDownloader { + async fn download_batch_mint( + &self, + url: &str, + ) -> Result, BatchMintValidationError>; + async fn download_batch_mint_and_check_checksum( + &self, + url: &str, + checksum: &str, + ) -> Result, BatchMintValidationError>; +} + +pub struct BatchMintPersister { + txn: Arc, + notification_receiver: Receiver<()>, + downloader: D, +} + +pub struct BatchMintDownloaderForPersister {} + +#[async_trait] +impl BatchMintDownloader for BatchMintDownloaderForPersister { + async fn download_batch_mint( + &self, + url: &str, + ) -> Result, BatchMintValidationError> { + let response = reqwest::get(url).await?.bytes().await?; + Ok(Box::new(serde_json::from_slice(&response)?)) + } + + async fn download_batch_mint_and_check_checksum( + &self, + url: &str, + checksum: &str, + ) -> Result, BatchMintValidationError> { + let response = reqwest::get(url).await?.bytes().await?; + let file_hash = xxhash_rust::xxh3::xxh3_128(&response); + let hash_hex = hex::encode(file_hash.to_be_bytes()); + if hash_hex != checksum { + return Err(BatchMintValidationError::InvalidDataHash( + checksum.to_string(), + hash_hex, + )); + } + Ok(Box::new(serde_json::from_slice(&response)?)) + } +} + +impl BatchMintPersister { + pub fn new(txn: Arc, notification_receiver: Receiver<()>, downloader: D) -> Self { + Self { + txn, + notification_receiver, + downloader, + } + } + + pub async fn persist_batch_mints(&self) { + loop { + if let Err(e) = self.notification_receiver.recv().await { + error!("Recv batch mint notification: {}", e); + continue; + }; + let Ok((batch_mint_to_verify, batch_mint)) = self.get_batch_mint_to_verify().await + else { + statsd_count!("batch_mint.fail_get_batch_mint", 1); + continue; + }; + let Some(batch_mint_to_verify) = batch_mint_to_verify else { + // no batch mints to persist + continue; + }; + let batch_mint = batch_mint + .map(|r| bincode::deserialize::(r.batch_mint_binary_bincode.as_slice())) + .transpose() + .unwrap_or_default(); + self.persist_batch_mint(batch_mint_to_verify, batch_mint.map(Box::new)) + .await; + } + } + + pub async fn persist_batch_mint( + &self, + mut batch_mint_to_verify: batch_mint_to_verify::Model, + mut batch_mint: Option>, + ) { + let start_time = Instant::now(); + info!("Persisting {} batch mint", &batch_mint_to_verify.url); + loop { + match &batch_mint_to_verify.batch_mint_persisting_state { + &BatchMintPersistingState::ReceivedTransaction => { + // We get ReceivedTransaction state on the start of processing + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::StartProcessing; + } + &BatchMintPersistingState::StartProcessing => { + if let Err(err) = self + .download_batch_mint(&mut batch_mint_to_verify, &mut batch_mint) + .await + { + error!("Error during batch mint downloading: {}", err) + }; + } + &BatchMintPersistingState::SuccessfullyDownload => { + if let Some(r) = &batch_mint { + self.validate_batch_mint(&mut batch_mint_to_verify, r).await; + } else { + error!( + "Trying to validate non downloaded batch mint: {:#?}", + &batch_mint_to_verify + ) + } + } + &BatchMintPersistingState::SuccessfullyValidate => { + if let Some(r) = &batch_mint { + if let Err(e) = self + .store_batch_mint_update(&mut batch_mint_to_verify, r) + .await + { + error!("Store batch mint update: {}", e) + }; + } else { + error!( + "Trying to store update for non downloaded batch mint: {:#?}", + &batch_mint_to_verify + ) + } + } + &BatchMintPersistingState::FailedToPersist + | &BatchMintPersistingState::StoredUpdate => { + if let Err(e) = self.drop_batch_mint_from_queue(&batch_mint_to_verify).await { + error!("failed to drop batch mint from queue: {}", e); + }; + info!( + "Finish processing {} batch mint file with {:?} state", + &batch_mint_to_verify.url, + &batch_mint_to_verify.batch_mint_persisting_state + ); + statsd_histogram!( + "batch_mint.persisting_latency", + start_time.elapsed().as_millis() as u64 + ); + statsd_count!("batch_mint.total_processed", 1); + return; + } + } + } + } + + pub async fn get_batch_mint_to_verify( + &self, + ) -> Result< + ( + Option, + Option, + ), + ProgramTransformerError, + > { + let multi_txn = self.txn.begin().await?; + let condition = Condition::all() + .add( + batch_mint_to_verify::Column::BatchMintPersistingState + .ne(BatchMintPersistingState::FailedToPersist), + ) + .add( + batch_mint_to_verify::Column::BatchMintPersistingState + .ne(BatchMintPersistingState::StoredUpdate), + ) + .add( + batch_mint_to_verify::Column::BatchMintPersistingState + .ne(BatchMintPersistingState::StartProcessing), + ); + + let batch_mint_verify = batch_mint_to_verify::Entity::find() + .filter(condition) + .order_by_asc(batch_mint_to_verify::Column::CreatedAtSlot) + .lock(LockType::Update) + .one(&multi_txn) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + let mut batch_mint = None; + if let Some(ref r) = batch_mint_verify { + batch_mint = batch_mint::Entity::find() + .filter(batch_mint::Column::FileHash.eq(r.file_hash.clone())) + .one(self.txn.as_ref()) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + batch_mint_to_verify::Entity::update(batch_mint_to_verify::ActiveModel { + file_hash: Set(r.file_hash.clone()), + url: Set(r.url.clone()), + created_at_slot: Set(r.created_at_slot), + signature: Set(r.signature.clone()), + merkle_tree: Set(r.merkle_tree.clone()), + staker: Set(r.staker.clone()), + download_attempts: Set(r.download_attempts), + batch_mint_persisting_state: Set(BatchMintPersistingState::StartProcessing), + batch_mint_fail_status: Set(r.batch_mint_fail_status.clone()), + collection: Set(r.collection.clone()), + }) + .filter(batch_mint_to_verify::Column::FileHash.eq(r.file_hash.clone())) + .exec(&multi_txn) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + } + multi_txn.commit().await?; + + Ok((batch_mint_verify, batch_mint)) + } + + async fn download_batch_mint( + &self, + batch_mint_to_verify: &mut batch_mint_to_verify::Model, + batch_mint: &mut Option>, + ) -> Result<(), ProgramTransformerError> { + if batch_mint.is_some() { + return Ok(()); + } + match self + .downloader + .download_batch_mint_and_check_checksum( + batch_mint_to_verify.url.as_ref(), + &batch_mint_to_verify.file_hash, + ) + .await + { + Ok(r) => { + let query = batch_mint::Entity::insert(batch_mint::ActiveModel { + file_hash: Set(batch_mint_to_verify.file_hash.clone()), + batch_mint_binary_bincode: Set(bincode::serialize(&r) + .map_err(|e| ProgramTransformerError::SerializatonError(e.to_string()))?), + }) + .on_conflict( + OnConflict::columns([batch_mint::Column::FileHash]) + .update_columns([batch_mint::Column::BatchMintBinaryBincode]) + .to_owned(), + ) + .build(DbBackend::Postgres); + if let Err(e) = self.txn.execute(query).await { + return Err(e.into()); + } + *batch_mint = Some(r); + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::SuccessfullyDownload; + statsd_count!("batch_mint.successfully_download", 1); + } + Err(e) => { + statsd_count!("batch_mint.download_fail", 1); + if let BatchMintValidationError::InvalidDataHash(expected, actual) = e { + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::FailedToPersist; + self.save_batch_mint_as_failed( + BatchMintFailStatus::ChecksumVerifyFailed, + batch_mint_to_verify, + ) + .await?; + + statsd_count!("batch_mint.checksum_verify_fail", 1); + return Err(ProgramTransformerError::BatchMintValidation( + BatchMintValidationError::FileChecksumMismatch(expected, actual), + )); + } + if let BatchMintValidationError::Serialization(e) = e { + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::FailedToPersist; + self.save_batch_mint_as_failed( + BatchMintFailStatus::FileSerialization, + batch_mint_to_verify, + ) + .await?; + + statsd_count!("batch_mint.file_deserialization_fail", 1); + return Err(ProgramTransformerError::SerializatonError(e)); + } + if batch_mint_to_verify.download_attempts + 1 + > MAX_BATCH_MINT_DOWNLOAD_ATTEMPTS as i32 + { + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::FailedToPersist; + self.save_batch_mint_as_failed( + BatchMintFailStatus::DownloadFailed, + batch_mint_to_verify, + ) + .await?; + } else { + batch_mint_to_verify.download_attempts = + batch_mint_to_verify.download_attempts + 1; + if let Err(e) = (batch_mint_to_verify::ActiveModel { + file_hash: Set(batch_mint_to_verify.file_hash.clone()), + url: Set(batch_mint_to_verify.url.clone()), + created_at_slot: Set(batch_mint_to_verify.created_at_slot), + signature: Set(batch_mint_to_verify.signature.clone()), + merkle_tree: Set(batch_mint_to_verify.merkle_tree.clone()), + staker: Set(batch_mint_to_verify.staker.clone()), + download_attempts: Set(batch_mint_to_verify.download_attempts + 1), + batch_mint_persisting_state: Set(batch_mint_to_verify + .batch_mint_persisting_state + .clone()), + batch_mint_fail_status: Set(batch_mint_to_verify + .batch_mint_fail_status + .clone()), + collection: Set(batch_mint_to_verify.collection.clone()), + } + .insert(self.txn.as_ref())) + .await + { + return Err(e.into()); + } + } + return Err(ProgramTransformerError::BatchMintValidation(e)); + } + } + Ok(()) + } + + async fn validate_batch_mint( + &self, + batch_mint_to_verify: &mut batch_mint_to_verify::Model, + batch_mint: &BatchMint, + ) { + let collection_raw_key: Option<[u8; 32]> = + if let Some(key) = batch_mint_to_verify.collection.clone() { + match key.try_into() { + Ok(key_as_array) => Some(key_as_array), + Err(e) => { + error!("Could not convert collection key received from DB: {:?}", e); + + self.mark_persisting_failed(batch_mint_to_verify).await; + return; + } + } + } else { + None + }; + + if let Err(e) = + validate_batch_mint(batch_mint, collection_raw_key.map(|key| Pubkey::from(key))).await + { + error!("Error while validating batch mint: {}", e.to_string()); + + self.mark_persisting_failed(batch_mint_to_verify).await; + return; + } + statsd_count!("batch_mint.validating_success", 1); + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::SuccessfullyValidate; + } + + async fn mark_persisting_failed(&self, batch_mint_to_verify: &mut batch_mint_to_verify::Model) { + statsd_count!("batch_mint.validating_fail", 1); + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::FailedToPersist; + if let Err(err) = self + .save_batch_mint_as_failed( + BatchMintFailStatus::BatchMintVerifyFailed, + batch_mint_to_verify, + ) + .await + { + error!("Save batch mint as failed: {}", err); + } + } + + async fn store_batch_mint_update( + &self, + batch_mint_to_verify: &mut batch_mint_to_verify::Model, + batch_mint: &BatchMint, + ) -> Result<(), ProgramTransformerError> { + if store_batch_mint_update( + batch_mint_to_verify.created_at_slot as u64, + batch_mint_to_verify.signature.clone(), + batch_mint, + self.txn.as_ref(), + ) + .await + .is_err() + { + statsd_count!("batch_mint.store_update_fail", 1); + batch_mint_to_verify.batch_mint_persisting_state = + BatchMintPersistingState::FailedToPersist; + return Ok(()); + } + statsd_count!("batch_mint.store_update_success", 1); + batch_mint_to_verify.batch_mint_persisting_state = BatchMintPersistingState::StoredUpdate; + Ok(()) + } + + async fn save_batch_mint_as_failed( + &self, + status: BatchMintFailStatus, + batch_mint: &batch_mint_to_verify::Model, + ) -> Result<(), ProgramTransformerError> { + batch_mint_to_verify::Entity::update(batch_mint_to_verify::ActiveModel { + file_hash: Set(batch_mint.file_hash.clone()), + url: Set(batch_mint.url.clone()), + created_at_slot: Set(batch_mint.created_at_slot), + signature: Set(batch_mint.signature.clone()), + merkle_tree: Set(batch_mint.merkle_tree.clone()), + staker: Set(batch_mint.staker.clone()), + download_attempts: Set(batch_mint.download_attempts), + batch_mint_persisting_state: Set(batch_mint.batch_mint_persisting_state.clone()), + batch_mint_fail_status: Set(Some(status)), + collection: Set(batch_mint.collection.clone()), + }) + .filter(batch_mint_to_verify::Column::FileHash.eq(batch_mint.file_hash.clone())) + .exec(self.txn.as_ref()) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + Ok(()) + } + + async fn drop_batch_mint_from_queue( + &self, + batch_mint_to_verify: &batch_mint_to_verify::Model, + ) -> Result<(), ProgramTransformerError> { + batch_mint_to_verify::Entity::update(batch_mint_to_verify::ActiveModel { + batch_mint_persisting_state: Set(batch_mint_to_verify + .batch_mint_persisting_state + .clone()), + ..batch_mint_to_verify.clone().into_active_model() + }) + .filter(batch_mint_to_verify::Column::FileHash.eq(batch_mint_to_verify.file_hash.clone())) + .exec(self.txn.as_ref()) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + + Ok(()) + } +} + +pub async fn store_batch_mint_update( + slot: u64, + signature: String, + batch_mint: &BatchMint, + txn: &T, +) -> ProgramTransformerResult<()> +where + T: ConnectionTrait + TransactionTrait, +{ + for batched_mint in batch_mint.batch_mints.iter() { + bubblegum::mint_v1::mint_v1( + &batched_mint.into(), + // only signature and slot will be used + &InstructionBundle { + txn_id: &signature, + slot, + ..Default::default() + }, + txn, + "FinalizeTreeWithRoot", + false, + ) + .await?; + } + + Ok(()) +} diff --git a/program_transformers/src/batch_minting/mod.rs b/program_transformers/src/batch_minting/mod.rs new file mode 100644 index 000000000..163029285 --- /dev/null +++ b/program_transformers/src/batch_minting/mod.rs @@ -0,0 +1 @@ +pub mod batch_mint_persister; diff --git a/program_transformers/src/bubblegum/db.rs b/program_transformers/src/bubblegum/db.rs index 08fb82f5f..02952abf2 100644 --- a/program_transformers/src/bubblegum/db.rs +++ b/program_transformers/src/bubblegum/db.rs @@ -1,5 +1,6 @@ use { crate::error::{ProgramTransformerError, ProgramTransformerResult}, + blockbuster::programs::bubblegum::ChangeLogEventV1, digital_asset_types::dao::{ asset, asset_authority, asset_creators, asset_data, asset_grouping, backfill_items, cl_audits_v2, cl_items, @@ -15,7 +16,6 @@ use { sea_query::query::OnConflict, ConnectionTrait, DbBackend, TransactionTrait, }, - spl_account_compression::events::ChangeLogEventV1, tracing::{debug, error, info}, }; diff --git a/program_transformers/src/bubblegum/finalize_tree_with_root.rs b/program_transformers/src/bubblegum/finalize_tree_with_root.rs new file mode 100644 index 000000000..96ef47cb7 --- /dev/null +++ b/program_transformers/src/bubblegum/finalize_tree_with_root.rs @@ -0,0 +1,73 @@ +use std::collections::HashSet; + +use blockbuster::programs::bubblegum::Payload; +use digital_asset_types::dao::sea_orm_active_enums::BatchMintPersistingState; +use sea_orm::sea_query::OnConflict; +use sea_orm::{DbBackend, EntityTrait, QueryTrait, Set}; +use solana_sdk::pubkey::Pubkey; +use tokio::sync::RwLock; +use { + crate::error::{ProgramTransformerError, ProgramTransformerResult}, + blockbuster::{instruction::InstructionBundle, programs::bubblegum::BubblegumInstruction}, + sea_orm::{ConnectionTrait, TransactionTrait}, +}; + +pub async fn finalize_tree_with_root<'c, T>( + parsing_result: &BubblegumInstruction, + bundle: &InstructionBundle<'c>, + txn: &'c T, + batched_trees: &Option>>, +) -> ProgramTransformerResult<()> +where + T: ConnectionTrait + TransactionTrait, +{ + if let Some(Payload::FinalizeTreeWithRoot { args, tree_id }) = &parsing_result.payload { + if let Some(batched_trees) = batched_trees { + let mut b_trees = batched_trees.write().await; + b_trees.insert(tree_id.clone()); + } + + let query = digital_asset_types::dao::batch_mint_to_verify::Entity::insert( + digital_asset_types::dao::batch_mint_to_verify::ActiveModel { + file_hash: Set(args.metadata_hash.clone()), + url: Set(args.metadata_url.clone()), + created_at_slot: Set(bundle.slot as i64), + signature: Set(bundle.txn_id.to_string()), + merkle_tree: Set(tree_id.to_bytes().to_vec()), + staker: Set(args.staker.to_bytes().to_vec()), + download_attempts: Set(0), + batch_mint_persisting_state: Set(BatchMintPersistingState::ReceivedTransaction), + batch_mint_fail_status: Set(None), + collection: Set(args.collection_mint.map(|k| k.to_bytes().to_vec())), + }, + ) + .on_conflict( + OnConflict::columns([ + digital_asset_types::dao::batch_mint_to_verify::Column::FileHash, + digital_asset_types::dao::batch_mint_to_verify::Column::Staker, + ]) + .update_columns([digital_asset_types::dao::batch_mint_to_verify::Column::Url]) + .update_columns([digital_asset_types::dao::batch_mint_to_verify::Column::Signature]) + .update_columns([ + digital_asset_types::dao::batch_mint_to_verify::Column::DownloadAttempts, + ]) + .update_columns([ + digital_asset_types::dao::batch_mint_to_verify::Column::BatchMintFailStatus, + ]) + .update_columns([ + digital_asset_types::dao::batch_mint_to_verify::Column::BatchMintPersistingState, + ]) + .update_columns([digital_asset_types::dao::batch_mint_to_verify::Column::CreatedAtSlot]) + .to_owned(), + ) + .build(DbBackend::Postgres); + txn.execute(query) + .await + .map_err(|e| ProgramTransformerError::DatabaseError(e.to_string()))?; + return Ok(()); + } + + Err(ProgramTransformerError::ParsingError( + "Ix not parsed correctly".to_string(), + )) +} diff --git a/program_transformers/src/bubblegum/mod.rs b/program_transformers/src/bubblegum/mod.rs index 07659011f..d3402bbb7 100644 --- a/program_transformers/src/bubblegum/mod.rs +++ b/program_transformers/src/bubblegum/mod.rs @@ -11,6 +11,9 @@ use { token_metadata::types::UseMethod as TokenMetadataUseMethod, }, sea_orm::{ConnectionTrait, TransactionTrait}, + solana_sdk::pubkey::Pubkey, + std::collections::HashSet, + tokio::sync::RwLock, tracing::{debug, info}, }; @@ -20,7 +23,8 @@ mod collection_verification; mod creator_verification; mod db; mod delegate; -mod mint_v1; +mod finalize_tree_with_root; +pub(crate) mod mint_v1; mod redeem; mod transfer; mod update_metadata; @@ -31,6 +35,7 @@ pub async fn handle_bubblegum_instruction<'c, T>( txn: &T, download_metadata_notifier: &DownloadMetadataNotifier, cl_audits: bool, + batched_trees: &Option>>, ) -> ProgramTransformerResult<()> where T: ConnectionTrait + TransactionTrait, @@ -58,6 +63,10 @@ where InstructionName::SetAndVerifyCollection => "SetAndVerifyCollection", InstructionName::SetDecompressibleState => "SetDecompressibleState", InstructionName::UpdateMetadata => "UpdateMetadata", + InstructionName::PrepareTree => "PrepareTree", + InstructionName::AddCanopy => "AddCanopy", + InstructionName::FinalizeTreeWithRoot => "FinalizeTreeWithRoot", + InstructionName::FinalizeTreeWithRootAndCollection => "FinalizeTreeWithRootAndCollection", }; info!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id); @@ -109,6 +118,16 @@ where .map_err(ProgramTransformerError::DownloadMetadataNotify)?; } } + InstructionName::FinalizeTreeWithRoot + | InstructionName::FinalizeTreeWithRootAndCollection => { + finalize_tree_with_root::finalize_tree_with_root( + parsing_result, + bundle, + txn, + batched_trees, + ) + .await? + } _ => debug!("Bubblegum: Not Implemented Instruction"), } Ok(()) diff --git a/program_transformers/src/error.rs b/program_transformers/src/error.rs index d0c29f383..5193b51f8 100644 --- a/program_transformers/src/error.rs +++ b/program_transformers/src/error.rs @@ -22,6 +22,8 @@ pub enum ProgramTransformerError { AssetIndexError(String), #[error("Failed to notify about download metadata: {0}")] DownloadMetadataNotify(Box), + #[error("BatchMintValidation: {0}")] + BatchMintValidation(#[from] BatchMintValidationError), } impl From for ProgramTransformerError { @@ -35,3 +37,61 @@ impl From for ProgramTransformerError { ProgramTransformerError::StorageWriteError(e.to_string()) } } + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum BatchMintValidationError { + #[error("PDACheckFail: expected: {0}, got: {1}")] + PDACheckFail(String, String), + #[error("InvalidDataHash: expected: {0}, got: {1}")] + InvalidDataHash(String, String), + #[error("InvalidCreatorsHash: expected: {0}, got: {1}")] + InvalidCreatorsHash(String, String), + #[error("InvalidRoot: expected: {0}, got: {1}")] + InvalidRoot(String, String), + #[error("NoRelevantRolledMint: index {0}")] + NoRelevantRolledMint(u64), + #[error("WrongAssetPath: id {0}")] + WrongAssetPath(String), + #[error("StdIo {0}")] + StdIo(String), + #[error("WrongTreeIdForChangeLog: asset: {0}, expected: {1}, got: {2}")] + WrongTreeIdForChangeLog(String, String, String), + #[error("WrongChangeLogIndex: asset: {0}, expected: {0}, got: {1}")] + WrongChangeLogIndex(String, u32, u32), + #[error("SplCompression: {0}")] + SplCompression(#[from] spl_account_compression::ConcurrentMerkleTreeError), + #[error("Anchor {0}")] + Anchor(#[from] anchor_lang::error::Error), + #[error("FileChecksumMismatch: expected {0}, actual file hash {1}")] + FileChecksumMismatch(String, String), + #[error("Unexpected tree depth={0} and max size={1}")] + UnexpectedTreeSize(u32, u32), + #[error("Serialization: {0}")] + Serialization(String), + #[error("Reqwest: {0}")] + Reqwest(String), + #[error("WrongCollectionVerified: {0}")] + WrongCollectionVerified(String), + #[error("VerifiedCollectionMismatch: expected :{0}, got :{1}")] + VerifiedCollectionMismatch(String, String), + #[error("Failed creator's signature verification: {0}")] + FailedCreatorVerification(String), + #[error("Missing creator's signature in batch mint: {0}")] + MissingCreatorSignature(String), +} + +impl From for BatchMintValidationError { + fn from(err: std::io::Error) -> Self { + BatchMintValidationError::StdIo(err.to_string()) + } +} +impl From for BatchMintValidationError { + fn from(value: serde_json::Error) -> Self { + Self::Serialization(value.to_string()) + } +} +impl From for BatchMintValidationError { + fn from(value: reqwest::Error) -> Self { + Self::Reqwest(value.to_string()) + } +} diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index edde20ca3..5620c68ca 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -15,20 +15,25 @@ use { ProgramParseResult, }, }, + digital_asset_types::dao::batch_mint_to_verify, futures::future::BoxFuture, sea_orm::{ entity::EntityTrait, query::Select, ConnectionTrait, DatabaseConnection, DbErr, - SqlxPostgresConnector, TransactionTrait, + QuerySelect, SqlxPostgresConnector, TransactionTrait, }, solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, solana_transaction_status::InnerInstructions, sqlx::PgPool, std::collections::{HashMap, HashSet, VecDeque}, - tokio::time::{sleep, Duration}, + tokio::{ + sync::RwLock, + time::{sleep, Duration}, + }, tracing::{debug, error, info}, }; mod asset_upserts; +pub mod batch_minting; mod bubblegum; pub mod error; mod mpl_core_program; @@ -85,13 +90,15 @@ pub struct ProgramTransformer { parsers: HashMap>, key_set: HashSet, cl_audits: bool, + batched_trees: Option>>, } impl ProgramTransformer { - pub fn new( + pub async fn new( pool: PgPool, download_metadata_notifier: DownloadMetadataNotifier, cl_audits: bool, + skip_batch_minted_trees: bool, ) -> Self { let mut parsers: HashMap> = HashMap::with_capacity(3); let bgum = BubblegumParser {}; @@ -107,12 +114,47 @@ impl ProgramTransformer { acc }); let pool: PgPool = pool; + + let storage = SqlxPostgresConnector::from_sqlx_postgres_pool(pool); + + let batched_trees = { + if !skip_batch_minted_trees { + None + } else { + match batch_mint_to_verify::Entity::find() + .column(batch_mint_to_verify::Column::MerkleTree) + .all(&storage) + .await + { + Ok(models) => { + let trees_result: Result, _> = models + .iter() + .map(|m| Pubkey::try_from(m.merkle_tree.clone())) + .collect(); + + match trees_result { + Ok(trees) => Some(RwLock::new(trees)), + Err(e) => { + error!("Failed to convert merkle_tree to Pubkey: {:?}", e); + None + } + } + } + Err(e) => { + error!("Failed to fetch batch_mint_to_verify models: {:?}", e); + None + } + } + } + }; + ProgramTransformer { - storage: SqlxPostgresConnector::from_sqlx_postgres_pool(pool), + storage, download_metadata_notifier, parsers, key_set: hs, cl_audits, + batched_trees, } } @@ -181,12 +223,23 @@ impl ProgramTransformer { let concrete = result.result_type(); match concrete { ProgramParseResult::Bubblegum(parsing_result) => { + if let Some(batched_trees) = &self.batched_trees { + if let Some(change_log) = &parsing_result.tree_update { + let batched_trees = batched_trees.read().await; + + if let Some(_tree) = batched_trees.get(&change_log.id) { + continue; + } + } + } + handle_bubblegum_instruction( parsing_result, &ix, &self.storage, &self.download_metadata_notifier, self.cl_audits, + &self.batched_trees, ) .await .map_err(|err| {