diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index fcf5c8e7a..0bd5fd326 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -33,3 +33,28 @@ jobs: crate-release-all: true pr-meta-comment: false pr-merge-strategy: "squash" + public-params: + if: inputs.bump == "major" + name: Generate & Upload PPs + runs-on: ['memory:32GB'] + environment: CI + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2024-11-09 + - name: Check aws is installed + run: aws --version + - uses: actions/setup-go@v5 + with: + go-version: '1.20' # The Go version to download (if necessary) and use. + - name: Generate PPs + run: | + ulimit -s unlimited + cargo run --release --bin gen-params + - name: Upload PPs + run: aws s3 cp --no-progress --recursive zkmr_params s3://lagrange-public-parameters --endpoint-url=https://428e47101872e479a0c311b576430fac.r2.cloudflarestorage.com + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: eeur diff --git a/Cargo.lock b/Cargo.lock index 0a24a2278..ed27cd244 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ dependencies = [ "alloy-rpc-types-eth", "alloy-sol-types", "alloy-transport", - "futures", + "futures 0.3.31", "futures-util", "thiserror 1.0.69", ] @@ -379,7 +379,7 @@ dependencies = [ "async-trait", "auto_impl", "dashmap 6.1.0", - "futures", + "futures 0.3.31", "futures-utils-wasm", "lru", "parking_lot", @@ -427,7 +427,7 @@ dependencies = [ "alloy-primitives 0.8.14", "alloy-transport", "alloy-transport-http", - "futures", + "futures 0.3.31", "pin-project", "reqwest 0.12.9", "serde", @@ -656,6 +656,15 @@ dependencies = [ "ansitok", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "ansitok" version = "0.2.0" @@ -851,6 +860,12 @@ version = "2.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "220a2c618ab466efe41d0eace94dfeff1c35e3aa47891bdb95e1c0fefffd3c99" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.5.2" @@ -911,11 +926,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ - "futures", + "futures 0.3.31", "pharos", "rustc_version 0.4.1", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atty" version = "0.2.14" @@ -1067,6 +1088,52 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee55e9ca33be1f257d8356cfb29b10b1c8f86dc38cf1344ca01525464356cd0c" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "blake2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +dependencies = [ + "crypto-mac", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec 0.7.6", + "cc", + "cfg-if", + "constant_time_eq 0.3.1", + "memmap2", + "rayon-core", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1076,6 +1143,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "blst" version = "0.3.13" @@ -1098,6 +1171,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + [[package]] name = "bumpalo" version = "3.16.0" @@ -1199,7 +1278,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" dependencies = [ "camino", "cargo-platform", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "thiserror 1.0.69", @@ -1222,6 +1301,35 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "checksums" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b400ece158d2e69b09054f6ce3a5773510e7cbde7effbc289c95041e19c3f5" +dependencies = [ + "blake", + "blake2", + "blake3", + "clap 2.34.0", + "crc", + "crc16", + "crc32c", + "crc8", + "futures 0.1.31", + "futures-cpupool", + "md-5 0.9.1", + "md6", + "num_cpus", + "once_cell", + "pbr", + "regex", + "shaman", + "tabwriter", + "tiny-keccak", + "walkdir", + "whirlpool", +] + [[package]] name = "chrono" version = "0.4.38" @@ -1245,6 +1353,21 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap", + "unicode-width 0.1.11", + "vec_map", +] + [[package]] name = "clap" version = "4.5.21" @@ -1264,7 +1387,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -1362,7 +1485,7 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", - "unicode-width", + "unicode-width 0.1.11", "windows-sys 0.52.0", ] @@ -1411,6 +1534,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -1442,6 +1571,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version 0.4.1", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1451,6 +1604,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc8" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b31d2174830f395fd7e413c2f8a119252de36356982f805f495269331e97559e" + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -1504,6 +1672,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "csv" version = "1.3.1" @@ -1594,7 +1772,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.11.1", "syn 2.0.89", ] @@ -1684,6 +1862,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-debug-plus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ff10b45018aea03566d0b5ae02e46372f482f9d23d756d1db5684a51b18a8a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -1747,7 +1936,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", "crypto-common", "subtle", @@ -2339,7 +2528,7 @@ dependencies = [ "chrono", "ethers-core 2.0.13", "reqwest 0.11.27", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "thiserror 1.0.69", @@ -2355,7 +2544,7 @@ dependencies = [ "chrono", "ethers-core 2.0.14", "reqwest 0.11.27", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "thiserror 1.0.69", @@ -2536,13 +2725,13 @@ dependencies = [ "ethers-core 2.0.13", "glob", "home", - "md-5", + "md-5 0.10.6", "num_cpus", "once_cell", "path-slash", "rayon", "regex", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "solang-parser", @@ -2568,13 +2757,13 @@ dependencies = [ "ethers-core 2.0.14", "glob", "home", - "md-5", + "md-5 0.10.6", "num_cpus", "once_cell", "path-slash", "rayon", "regex", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "solang-parser", @@ -2754,6 +2943,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.31" @@ -2785,6 +2980,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures 0.1.31", + "num_cpus", +] + [[package]] name = "futures-executor" version = "0.3.31" @@ -2976,6 +3181,17 @@ dependencies = [ "gobuild", ] +[[package]] +name = "gnark-utils" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "anyhow", + "base64 0.22.1", + "glob", + "gobuild", +] + [[package]] name = "gobuild" version = "0.1.0-alpha.2" @@ -2992,22 +3208,40 @@ dependencies = [ "alloy", "anyhow", "env_logger 0.11.5", - "gnark-utils", + "gnark-utils 1.2.1", "hex", "itertools 0.13.0", "log", - "mp2_common", - "mp2_test", + "mp2_common 1.2.1", + "mp2_test 1.2.1", "plonky2", "plonky2x", "rand", - "recursion_framework", + "recursion_framework 1.2.1", "revm", "serde", "serde_json", "serial_test 3.2.0", "sha2", - "verifiable-db", + "verifiable-db 1.2.1", +] + +[[package]] +name = "groth16_framework" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "alloy", + "anyhow", + "gnark-utils 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "hex", + "log", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "plonky2", + "plonky2x", + "revm", + "serde", + "serde_json", ] [[package]] @@ -3040,6 +3274,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.6.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3235,7 +3488,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -3258,6 +3511,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -3277,9 +3531,26 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.31", - "rustls", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.5.1", + "hyper-util", + "rustls 0.23.22", + "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.1", + "tower-service", ] [[package]] @@ -3578,13 +3849,13 @@ name = "inspect" version = "1.2.1" dependencies = [ "anyhow", - "clap", + "clap 4.5.21", "colored", "dialoguer", "hex", "itertools 0.13.0", - "mp2_v1", - "ryhope", + "mp2_v1 1.2.1", + "ryhope 1.2.1", "serde", "tabled", "tokio", @@ -3795,6 +4066,48 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "lgn-messages" +version = "1.1.2" +source = "git+https://github.com/Lagrange-Labs/lgn-coprocessor.git?branch=main#55e8c7715fbe260ba90c5e026f9fc5f5d9ea70e5" +dependencies = [ + "alloy-primitives 0.8.14", + "derive-debug-plus", + "ethers 2.0.14", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "mp2_v1 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "object_store", + "serde", + "serde_derive", + "thiserror 1.0.69", + "verifiable-db 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", +] + +[[package]] +name = "lgn-provers" +version = "1.1.2" +source = "git+https://github.com/Lagrange-Labs/lgn-coprocessor.git?branch=main#55e8c7715fbe260ba90c5e026f9fc5f5d9ea70e5" +dependencies = [ + "alloy", + "anyhow", + "bincode", + "bytes", + "checksums", + "ethers 2.0.14", + "groth16_framework 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "lgn-messages", + "metrics", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "mp2_v1 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "parsil 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "rand", + "reqwest 0.12.9", + "serde", + "serde_json", + "tracing", + "verifiable-db 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", +] + [[package]] name = "libc" version = "0.2.167" @@ -3863,6 +4176,17 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "md-5" version = "0.10.6" @@ -3873,6 +4197,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md6" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed426378c4745d1257e49ee3f729aaec2f0c6f29113ab7835a0b5938ca07b86f" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.7.4" @@ -3888,6 +4222,16 @@ dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" +dependencies = [ + "ahash", + "portable-atomic", +] + [[package]] name = "mime" version = "0.3.17" @@ -3936,7 +4280,7 @@ dependencies = [ "hex", "itertools 0.13.0", "log", - "mp2_test", + "mp2_test 1.2.1", "num", "plonky2", "plonky2_crypto", @@ -3951,21 +4295,68 @@ dependencies = [ "tokio", ] +[[package]] +name = "mp2_common" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "alloy", + "anyhow", + "bincode", + "derive_more 1.0.0", + "eth_trie", + "ethereum-types", + "git-version", + "hashbrown 0.15.2", + "hex", + "itertools 0.13.0", + "log", + "num", + "plonky2", + "plonky2_crypto", + "plonky2_ecdsa", + "plonky2_ecgfp5", + "poseidon2_plonky2", + "rand", + "rlp", + "serde", + "sha3", +] + +[[package]] +name = "mp2_test" +version = "1.2.1" +dependencies = [ + "alloy", + "anyhow", + "env_logger 0.11.5", + "eth_trie", + "log", + "mp2_common 1.2.1", + "plonky2", + "plonky2_ecgfp5", + "rand", + "recursion_framework 1.2.1", + "ryhope 1.2.1", + "serde", +] + [[package]] name = "mp2_test" version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", "env_logger 0.11.5", "eth_trie", "log", - "mp2_common", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", "plonky2", "plonky2_ecgfp5", "rand", - "recursion_framework", - "ryhope", + "recursion_framework 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "ryhope 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", "serde", ] @@ -3983,25 +4374,25 @@ dependencies = [ "env_logger 0.11.5", "envconfig", "eth_trie", - "futures", + "futures 0.3.31", "hashbrown 0.15.2", "hex", "itertools 0.13.0", "jammdb", "lazy_static", "log", - "mp2_common", - "mp2_test", - "parsil", + "mp2_common 1.2.1", + "mp2_test 1.2.1", + "parsil 1.2.1", "paste", "plonky2", "plonky2_crypto", "plonky2_ecgfp5", "rand", "rand_chacha", - "recursion_framework", + "recursion_framework 1.2.1", "rlp", - "ryhope", + "ryhope 1.2.1", "serde", "serde_json", "serial_test 3.2.0", @@ -4011,7 +4402,41 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", - "verifiable-db", + "verifiable-db 1.2.1", +] + +[[package]] +name = "mp2_v1" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "alloy", + "anyhow", + "bb8", + "bb8-postgres", + "bincode", + "derive_more 1.0.0", + "eth_trie", + "futures 0.3.31", + "hashbrown 0.15.2", + "hex", + "itertools 0.13.0", + "log", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "parsil 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "paste", + "plonky2", + "plonky2_crypto", + "plonky2_ecgfp5", + "rand", + "recursion_framework 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "rlp", + "ryhope 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "serde", + "serde_json", + "tokio-postgres", + "tracing", + "verifiable-db 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", ] [[package]] @@ -4190,12 +4615,39 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures 0.3.31", + "humantime", + "itertools 0.13.0", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open-fastrlp" version = "0.1.4" @@ -4297,7 +4749,7 @@ dependencies = [ "ansitok", "bytecount", "fnv", - "unicode-width", + "unicode-width 0.1.11", ] [[package]] @@ -4357,15 +4809,32 @@ dependencies = [ "alloy", "anyhow", "camelpaste", - "clap", + "clap 4.5.21", "log", - "ryhope", + "ryhope 1.2.1", "serde", "serde_json", "sqlparser", "stderrlog", "thiserror 2.0.3", - "verifiable-db", + "verifiable-db 1.2.1", +] + +[[package]] +name = "parsil" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "alloy", + "anyhow", + "camelpaste", + "log", + "ryhope 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "serde", + "serde_json", + "sqlparser", + "thiserror 2.0.3", + "verifiable-db 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", ] [[package]] @@ -4413,6 +4882,17 @@ dependencies = [ "hmac", ] +[[package]] +name = "pbr" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5827dfa0d69b6c92493d6c38e633bbaa5937c153d0d7c28bf12313f8c6d514" +dependencies = [ + "crossbeam-channel", + "libc", + "winapi", +] + [[package]] name = "pem" version = "1.1.1" @@ -4455,7 +4935,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ - "futures", + "futures 0.3.31", "rustc_version 0.4.1", ] @@ -4696,7 +5176,7 @@ dependencies = [ "backtrace", "base64 0.13.1", "bincode", - "clap", + "clap 4.5.21", "curve25519-dalek 3.2.0", "digest 0.10.7", "dotenv", @@ -4704,7 +5184,7 @@ dependencies = [ "env_logger 0.10.2", "ethers 2.0.14", "ff", - "futures", + "futures 0.3.31", "hex", "itertools 0.10.5", "lazy_static", @@ -4737,6 +5217,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "poseidon2_plonky2" version = "0.1.0" @@ -4760,7 +5246,7 @@ dependencies = [ "bytes", "fallible-iterator", "hmac", - "md-5", + "md-5 0.10.6", "memchr", "rand", "sha2", @@ -5018,7 +5504,7 @@ dependencies = [ "bincode", "env_logger 0.11.5", "log", - "mp2_common", + "mp2_common 1.2.1", "plonky2", "plonky2_monolith", "rstest 0.23.0", @@ -5026,6 +5512,18 @@ dependencies = [ "serial_test 3.2.0", ] +[[package]] +name = "recursion_framework" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "anyhow", + "log", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "plonky2", + "serde", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -5107,11 +5605,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.31", - "hyper-rustls", + "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", "js-sys", @@ -5121,16 +5619,16 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -5148,12 +5646,16 @@ checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", + "futures-channel", "futures-core", "futures-util", + "h2 0.4.7", "http 1.1.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", + "hyper-rustls 0.27.5", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -5169,6 +5671,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.2", + "system-configuration 0.6.1", "tokio", "tokio-native-tls", "tower-service", @@ -5307,7 +5810,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" dependencies = [ - "futures", + "futures 0.3.31", "futures-timer", "rstest_macros 0.17.0", "rustc_version 0.4.1", @@ -5319,7 +5822,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" dependencies = [ - "futures", + "futures 0.3.31", "futures-timer", "rstest_macros 0.23.0", "rustc_version 0.4.1", @@ -5407,6 +5910,12 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc-serialize" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" + [[package]] name = "rustc_version" version = "0.3.3" @@ -5422,7 +5931,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.25", ] [[package]] @@ -5446,10 +5955,23 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.8", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -5484,6 +6006,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.18" @@ -5510,7 +6043,7 @@ dependencies = [ "bb8", "bb8-postgres", "delegate", - "futures", + "futures 0.3.31", "hex", "itertools 0.13.0", "log", @@ -5526,6 +6059,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "ryhope" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "anyhow", + "bb8", + "bb8-postgres", + "delegate", + "futures 0.3.31", + "hex", + "itertools 0.13.0", + "log", + "postgres-types", + "serde", + "serde_json", + "thiserror 2.0.3", + "tokio", + "tokio-postgres", + "tracing", +] + [[package]] name = "ryu" version = "1.0.18" @@ -5685,9 +6240,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -5840,7 +6395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "538c30747ae860d6fb88330addbbd3e0ddbe46d662d032855596d8a8ca260611" dependencies = [ "dashmap 5.5.3", - "futures", + "futures 0.3.31", "lazy_static", "log", "parking_lot", @@ -5853,7 +6408,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" dependencies = [ - "futures", + "futures 0.3.31", "log", "once_cell", "parking_lot", @@ -5938,6 +6493,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "shaman" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1a48b7e97a07b8f5771159e6914d8459557f8f833a60c4bd041f03736844f5" +dependencies = [ + "rand", + "rustc-serialize", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -6022,6 +6587,27 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "socket2" version = "0.5.8" @@ -6145,6 +6731,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.11.1" @@ -6212,7 +6804,7 @@ dependencies = [ "hex", "once_cell", "reqwest 0.11.27", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "sha2", @@ -6289,7 +6881,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -6302,6 +6905,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tabled" version = "0.16.0" @@ -6327,6 +6940,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "tabwriter" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" +dependencies = [ + "unicode-width 0.2.0", +] + [[package]] name = "tap" version = "1.0.1" @@ -6397,6 +7019,15 @@ dependencies = [ "rand", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.11", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -6592,7 +7223,17 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls 0.23.22", "tokio", ] @@ -6616,9 +7257,9 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tungstenite", "webpki-roots", ] @@ -6670,6 +7311,23 @@ dependencies = [ "winnow", ] +[[package]] +name = "tools" +version = "1.2.1" +dependencies = [ + "bincode", + "blake3", + "clap 4.5.21", + "groth16_framework 1.2.1", + "lgn-messages", + "lgn-provers", + "mp2_v1 1.2.1", + "semver 1.0.25", + "tokio", + "verifiable-db 1.2.1", + "walkdir", +] + [[package]] name = "tower" version = "0.5.1" @@ -6785,7 +7443,7 @@ dependencies = [ "httparse", "log", "rand", - "rustls", + "rustls 0.21.12", "sha1", "thiserror 1.0.69", "url", @@ -6855,6 +7513,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -6949,6 +7613,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "verifiable-db" version = "1.2.1" @@ -6957,23 +7627,46 @@ dependencies = [ "anyhow", "bincode", "derive_more 1.0.0", - "futures", + "futures 0.3.31", "itertools 0.13.0", "log", - "mp2_common", - "mp2_test", + "mp2_common 1.2.1", + "mp2_test 1.2.1", "plonky2", "plonky2_crypto", "plonky2_ecdsa", "plonky2_ecgfp5", "rand", - "recursion_framework", - "ryhope", + "recursion_framework 1.2.1", + "ryhope 1.2.1", "serde", "serial_test 3.2.0", "tokio", ] +[[package]] +name = "verifiable-db" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" +dependencies = [ + "alloy", + "anyhow", + "bincode", + "derive_more 1.0.0", + "itertools 0.13.0", + "log", + "mp2_common 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "mp2_test 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "plonky2", + "plonky2_crypto", + "plonky2_ecdsa", + "plonky2_ecgfp5", + "rand", + "recursion_framework 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "ryhope 1.2.1 (git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1)", + "serde", +] + [[package]] name = "version_check" version = "0.9.5" @@ -7121,7 +7814,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0048ad49a55b9deb3953841fa1fc5858f0efbcb7a18868c899a360269fac1b23" dependencies = [ - "futures", + "futures 0.3.31", "js-sys", "parking_lot", "pin-utils", @@ -7155,6 +7848,17 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "whirlpool" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6b887d628151d048b725717536b20888cdf177c5673a5f5dfc48d700272664" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "whoami" version = "1.5.2" @@ -7422,7 +8126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" dependencies = [ "async_io_stream", - "futures", + "futures 0.3.31", "js-sys", "log", "pharos", @@ -7566,7 +8270,7 @@ dependencies = [ "aes", "byteorder", "bzip2", - "constant_time_eq", + "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", "flate2", diff --git a/Cargo.toml b/Cargo.toml index d6af4c21f..cc0c99961 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "parsil", "ryhope", "recursion-framework", + "tools", "verifiable-db", ] resolver = "2" diff --git a/tools/Cargo.toml b/tools/Cargo.toml new file mode 100644 index 000000000..694d8f418 --- /dev/null +++ b/tools/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "tools" +version = "1.2.1" +edition = "2021" + +[[bin]] +name = "gen-params" +path = "src/gen-params.rs" + +[dependencies] +bincode.workspace = true +blake3 = { version = "1.5.5", features = ["mmap", "rayon"] } +clap = { workspace = true, features = ["derive", "std", "suggestions", "help"] } +groth16_framework.path = "../groth16-framework/" +lgn-messages = { git = "https://github.com/Lagrange-Labs/lgn-coprocessor.git", branch = "main" } +lgn-provers = { git = "https://github.com/Lagrange-Labs/lgn-coprocessor.git", branch = "main" } +mp2_v1.path = "../mp2-v1/" +semver = "1.0.25" +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +verifiable-db.path = "../verifiable-db/" +walkdir = "2.5.0" diff --git a/tools/src/gen-params.rs b/tools/src/gen-params.rs new file mode 100755 index 000000000..667e1d345 --- /dev/null +++ b/tools/src/gen-params.rs @@ -0,0 +1,243 @@ +#![feature(generic_const_exprs)] +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use clap::Parser; +use groth16_framework::compile_and_generate_assets; +use groth16_framework::utils::clone_circuit_data; +use mp2_v1::api::{build_circuits_params, PublicParameters}; +use verifiable_db::api::QueryParameters; + +use lgn_messages::types::v1::query::{NUM_CHUNKS, NUM_ROWS}; +use lgn_provers::params::PARAMS_CHECKSUM_FILENAME; +use lgn_provers::provers::v1::query::MAX_NUM_OUTPUTS; +use lgn_provers::provers::v1::query::MAX_NUM_PLACEHOLDERS; +use lgn_provers::provers::v1::query::MAX_NUM_PREDICATE_OPS; +use lgn_provers::provers::v1::query::MAX_NUM_RESULT_OPS; +use lgn_provers::provers::v1::query::{INDEX_TREE_MAX_DEPTH, MAX_NUM_ITEMS_PER_OUTPUT}; +use lgn_provers::provers::v1::query::{MAX_NUM_COLUMNS, ROW_TREE_MAX_DEPTH}; + +const GROTH16_ASSETS_PREFIX: &str = "groth16_assets"; +const PP_BIN_KEY: &str = "preprocessing_params.bin"; +const QP_BIN_KEY: &str = "query_params.bin"; + +type QueryParams = QueryParameters< + NUM_CHUNKS, + NUM_ROWS, + ROW_TREE_MAX_DEPTH, + INDEX_TREE_MAX_DEPTH, + MAX_NUM_COLUMNS, + MAX_NUM_PREDICATE_OPS, + MAX_NUM_RESULT_OPS, + MAX_NUM_OUTPUTS, + MAX_NUM_ITEMS_PER_OUTPUT, + MAX_NUM_PLACEHOLDERS, +>; +type PreprocessingParameters = PublicParameters; + +/// Generate the public parameters for the current version of MR2. +#[derive(Debug, Parser)] +struct Args { + /// Where to serialize the public parameters. + #[arg(short, long, default_value = "./zkmr_params")] + params_root_dir: String, + + /// Generate Groth16 parameters from existing query parameters + #[arg(long)] + only_groth16: bool, +} + +/// Build the params directory with a sub-path of mp2 version. +fn params_dir(params_dir: &str) -> String { + let path = Path::new(params_dir); + let mp2_version_str = verifiable_db::version(); + let version = semver::Version::parse(mp2_version_str).unwrap(); + + path.join(version.major.to_string()) + .to_string_lossy() + .to_string() +} + +/// Given a config, walk its PPs direct and write the list of the relative path +/// of the contained files and their B3 hash to the target hash file. +fn write_hashes(params_root_dir: &str) { + let hash_file_path = + Path::new(params_dir(params_root_dir).as_str()).join(PARAMS_CHECKSUM_FILENAME); + let mut out_file = File::create(hash_file_path).expect("failed to create hash file"); + + for entry in walkdir::WalkDir::new(params_root_dir) + .into_iter() + .filter_map(|e| e.ok()) + .map(|e| e.path().to_path_buf()) + .filter(|p| p.is_file()) + { + println!("hashing {}", entry.display()); + let mut hasher = blake3::Hasher::new(); + hasher + .update_mmap_rayon(entry.as_path()) + .expect("hashing failed"); + let hash_str = hasher.finalize().to_hex(); + out_file + .write_all( + format!( + "{} {}\n", + entry + .strip_prefix(params_dir(params_root_dir)) + .unwrap() + .display(), + hash_str + ) + .as_bytes(), + ) + .expect("failed to write to hash file"); + } +} + +/// Main entry point for the parameter generation tool +#[tokio::main] +async fn main() { + // Parse the CLI arguments. + let args = Args::parse(); + + println!("serializing parameters to `{}`", args.params_root_dir); + + let query_params = if args.only_groth16 { + load_query_params_from_disk(&args.params_root_dir) + } else { + // TRICKY: The parameters have large size, we suppose to generate and drop it in a local + // scope to avoid stack overflow, and also need to avoid passing into an async function. + + let preprocessing_params = build_preprocessing_params(); + let query_params = build_query_parameters(&preprocessing_params); + generate_groth16_assets(&args.params_root_dir, &query_params); + + let _ = store_preprocessing_params(&args.params_root_dir, &preprocessing_params); + let _ = store_query_params(&args.params_root_dir, &query_params); + + query_params + }; + generate_groth16_assets(&args.params_root_dir, &query_params); + + write_hashes(&args.params_root_dir); +} + +/// Build preprocessing parameters +fn build_preprocessing_params() -> PreprocessingParameters { + let now = Instant::now(); + + println!("Start to generate the preprocessing parameters"); + + let params = build_circuits_params(); + + println!( + "Finish generating the preprocessing parameters, elapsed: {:?}", + now.elapsed() + ); + + params +} + +/// Store preprocessing parameters on disk and return the saved file path +fn store_preprocessing_params( + params_root_dir: &str, + preprocessing_params: &PreprocessingParameters, +) -> PathBuf { + let _now = Instant::now(); + + // Serialize the preprocessing parameters. + println!("Start to serialize the preprocessing parameters"); + let data = bincode::serialize(&preprocessing_params).unwrap(); + println!("Finish serializing the preprocessing parameters"); + + // Store on disk + println!("Start to store the preprocessing parameters on disk"); + + let file_path = Path::new(params_dir(params_root_dir).as_str()).join(PP_BIN_KEY); + println!("Writing to file: {:?}", file_path); + + // Try to create the parent dir if not exists. + if let Some(parent_dir) = file_path.parent() { + std::fs::create_dir_all(parent_dir).expect("Parent dirs should be created"); + } + + let mut buffer = File::create(file_path.clone()).unwrap(); + buffer.write_all(&data).unwrap(); + println!("Finish storing the preprocessing parameters on disk"); + + file_path +} + +/// Build query parameters from preprocessing parameters +fn build_query_parameters(indexing_params: &PublicParameters) -> QueryParams { + QueryParameters::build_params(&indexing_params.get_params_info().unwrap()).unwrap() +} + +/// Store query parameters on disk and return the saved file path +fn store_query_params(params_root_dir: &str, query_params: &QueryParams) -> PathBuf { + let _now = Instant::now(); + + // Serialize the preprocessing parameters. + println!("Start to serialize the query parameters"); + let data = bincode::serialize(query_params).unwrap(); + println!("Finish serializing the query parameters"); + + // Store on disk + println!("Start to store the query parameters on disk"); + let file_path = Path::new(params_dir(params_root_dir).as_str()).join(QP_BIN_KEY); + println!("Writing to file: {:?}", file_path); + + // Try to create the parent dir if not exists. + if let Some(parent_dir) = file_path.parent() { + std::fs::create_dir_all(parent_dir).expect("Parent dirs should be created"); + } + + let mut buffer = File::create(file_path.clone()).unwrap(); + buffer.write_all(&data).unwrap(); + println!("Finish storing the query parameters on disk"); + + file_path +} + +/// Load query parameters from disk +fn load_query_params_from_disk(params_root_dir: &str) -> QueryParams { + let now = Instant::now(); + println!("Start loading query parameters from disk"); + + let file_path = Path::new(params_dir(params_root_dir).as_str()).join(QP_BIN_KEY); + let mut file = File::open(file_path).unwrap_or_else(|_| panic!("Failed to open {QP_BIN_KEY}")); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).expect("Failed to read file"); + + let query_params: QueryParams = + bincode::deserialize(&buffer).expect("Failed to deserialize query parameters"); + + println!( + "Finished loading query parameters from disk, elapsed: {:?}", + now.elapsed() + ); + + query_params +} + +/// Generate Groth16 asset files and save to disk +fn generate_groth16_assets(params_root_dir: &str, query_params: &QueryParams) { + let now = Instant::now(); + + println!("Start to generate the Groth16 asset files"); + + // Get the final circuit data of the query parameters. + let circuit_data = query_params.final_proof_circuit_data(); + let circuit_data = clone_circuit_data(circuit_data) + .unwrap_or_else(|err| panic!("Failed to clone the circuit data: {}", err)); + + // Compile and generate the Groth16 asset files. + let assets_dir = format!("{}/{GROTH16_ASSETS_PREFIX}", params_dir(params_root_dir)); + compile_and_generate_assets(circuit_data, &assets_dir).unwrap(); + + println!( + "Finish generating the Groth16 asset files, elapsed: {:?}", + now.elapsed() + ); +}