diff --git a/.gitignore b/.gitignore index d70d2739..eb37e169 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,8 @@ coverage.json logs/ chaindata/ ordhook-data/ -db/ +db redis-data/ blocks/ +qbench_data/ hints.json diff --git a/Cargo.lock b/Cargo.lock index fb1d7d04..e188fa0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -168,6 +183,22 @@ dependencies = [ "term", ] +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd 0.13.2", + "zstd-safe 7.2.0", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -281,15 +312,6 @@ version = "0.10.0-beta" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98f7eed2b2781a6f0b5c903471d48e15f56fb4e1165df8a9a2337fd1a59d45ea" -[[package]] -name = "beef" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] - [[package]] name = "bincode" version = "1.3.3" @@ -299,27 +321,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.65.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" -dependencies = [ - "bitflags 1.2.1", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.64", -] - [[package]] name = "bindgen" version = "0.69.4" @@ -337,7 +338,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.64", "which", @@ -443,20 +444,32 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "brotli" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ - "generic-array", + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -566,6 +579,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -645,6 +664,7 @@ dependencies = [ "city_rollup_core_node", "city_rollup_core_orchestrator", "city_rollup_core_worker", + "city_rollup_core_worker_qbench", "city_store", "clap", "dotenv", @@ -711,6 +731,7 @@ dependencies = [ "city_rollup_core_node", "city_rollup_core_orchestrator", "city_rollup_rpc_provider", + "city_rollup_user_prover_api", "city_store", "clap", "dotenv", @@ -902,12 +923,13 @@ dependencies = [ "anyhow", "city_common", "city_crypto", + "city_macros", "city_rollup_common", "city_store", "futures", "hex", "jsonrpsee", - "kvq_store_rocksdb", + "kvq_store_redb", "redb", "tokio", ] @@ -945,6 +967,8 @@ dependencies = [ "serde_json", "serde_with 3.8.1", "tokio", + "tower", + "tower-http", "tracing", "url", ] @@ -972,10 +996,11 @@ dependencies = [ "hex-literal", "k256", "kvq", - "kvq_store_rocksdb", + "kvq_store_redb", "plonky2", "rand", "rand_chacha", + "redb", "redis", "reqwest", "serde", @@ -1021,12 +1046,50 @@ dependencies = [ "url", ] +[[package]] +name = "city_rollup_core_worker_qbench" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "bitcoin", + "city_common", + "city_common_circuit", + "city_crypto", + "city_macros", + "city_redis_store", + "city_rollup_circuit", + "city_rollup_common", + "city_rollup_core_orchestrator", + "city_rollup_core_worker", + "city_rollup_worker_dispatch", + "city_store", + "criterion", + "crossterm", + "gnark-plonky2-wrapper", + "hex", + "hex-literal", + "home", + "k256", + "plonky2", + "rand", + "rand_chacha", + "reqwest", + "serde", + "serde_json", + "serde_with 3.8.1", + "tokio", + "tracing", + "url", +] + [[package]] name = "city_rollup_rpc_provider" version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "city_common", "city_crypto", "city_macros", "city_rollup_common", @@ -1038,6 +1101,30 @@ dependencies = [ "serde_json", ] +[[package]] +name = "city_rollup_user_prover_api" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "city_common", + "city_common_circuit", + "city_crypto", + "city_macros", + "city_rollup_circuit", + "city_rollup_common", + "futures", + "hex", + "hyper 1.3.1", + "jsonrpsee", + "plonky2", + "redb", + "serde", + "tokio", + "tower", + "tower-http", +] + [[package]] name = "city_rollup_worker_dispatch" version = "0.1.0" @@ -1177,7 +1264,7 @@ checksum = "3b6be4a5df2098cd811f3194f64ddb96c267606bffd9689ac7b0160097b01ad3" dependencies = [ "bs58", "coins-core", - "digest 0.10.7", + "digest", "hmac", "k256", "serde", @@ -1210,7 +1297,7 @@ dependencies = [ "base64 0.21.7", "bech32 0.9.1", "bs58", - "digest 0.10.7", + "digest", "generic-array", "hex", "ripemd", @@ -1474,7 +1561,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", - "digest 0.10.7", + "digest", "fiat-crypto", "platforms", "rustc_version", @@ -1565,22 +1652,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "const-oid", "crypto-common", "subtle", @@ -1647,7 +1725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der", - "digest 0.10.7", + "digest", "elliptic-curve", "rfc6979", "serdect", @@ -1694,7 +1772,7 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "digest 0.10.7", + "digest", "ff", "generic-array", "group", @@ -1814,7 +1892,7 @@ checksum = "1fda3bf123be441da5260717e0661c25a2fd9cb2b2c1d20bf2e05580047158ab" dependencies = [ "aes", "ctr", - "digest 0.10.7", + "digest", "hex", "hmac", "pbkdf2 0.11.0", @@ -2487,7 +2565,7 @@ version = "0.1.0" source = "git+https://github.com/cf/gnark-plonky2-verifier?rev=75681e2a0e20270734f082145d13cdfe3349d009#75681e2a0e20270734f082145d13cdfe3349d009" dependencies = [ "anyhow", - "bindgen 0.69.4", + "bindgen", "gobuild", "libc", ] @@ -2612,6 +2690,16 @@ dependencies = [ "fxhash", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -2669,7 +2757,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2737,6 +2825,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" + [[package]] name = "httparse" version = "1.8.0" @@ -2809,13 +2903,29 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.28", - "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.3.1", + "hyper-util", + "log", + "rustls 0.23.11", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2978,6 +3088,16 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "iri-string" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f5f6c2df22c009ac44f6f1499308e7a3ac7ba42cd2378475cc691510e1eef1b" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.12" @@ -3034,6 +3154,26 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.31" @@ -3065,9 +3205,9 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdb12a2381ea5b2e68c3469ec604a007b367778cdb14d09612c8069ebd616ad" +checksum = "e419d6c39cb9632288c592a06d7d0a96740021b0bff812e211ace754b0fe8c9a" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -3083,45 +3223,47 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4978087a58c3ab02efc5b07c5e5e2803024536106fd5506f558db172c889b3aa" +checksum = "f8b6dc3b1e2da087969e69a095e7d6127966c8c194490b311017bb2053a7d5a1" dependencies = [ + "base64 0.22.1", "futures-channel", "futures-util", "gloo-net", - "http 0.2.12", + "http 1.1.0", "jsonrpsee-core", "pin-project", - "rustls-native-certs 0.7.0", + "rustls 0.23.11", "rustls-pki-types", + "rustls-platform-verifier", "soketto", "thiserror", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-util", "tracing", "url", - "webpki-roots 0.26.1", ] [[package]] name = "jsonrpsee-core" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b257e1ec385e07b0255dde0b933f948b5c8b8c28d42afda9587c3a967b896d" +checksum = "d06b8be79a3bdd7d87c1d95c0e939052b7f64fffce7b9436986e43e92f20a978" dependencies = [ - "anyhow", "async-trait", - "beef", + "bytes", "futures-timer", "futures-util", - "hyper 0.14.28", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "jsonrpsee-types", "parking_lot", "pin-project", "rand", - "rustc-hash", + "rustc-hash 2.0.0", "serde", "serde_json", "thiserror", @@ -3133,15 +3275,20 @@ dependencies = [ [[package]] name = "jsonrpsee-http-client" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccf93fc4a0bfe05d851d37d7c32b7f370fe94336b52a2f0efc5f1981895c2e5" +checksum = "52dc99c70619e252e6adc5e95144323505a69a1742771de5b3f2071e1595b363" dependencies = [ "async-trait", - "hyper 0.14.28", - "hyper-rustls", + "base64 0.22.1", + "http-body 1.0.0", + "hyper 1.3.1", + "hyper-rustls 0.27.2", + "hyper-util", "jsonrpsee-core", "jsonrpsee-types", + "rustls 0.23.11", + "rustls-platform-verifier", "serde", "serde_json", "thiserror", @@ -3153,11 +3300,11 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d0bb047e79a143b32ea03974a6bf59b62c2a4c5f5d42a381c907a8bbb3f75c0" +checksum = "b4278d453682d9f9671b5261404360cdabd063198a43bb221c5c80e8f8bfb6b1" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro-crate", "proc-macro2", "quote", @@ -3166,13 +3313,16 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d8b6a9674422a8572e0b0abb12feeb3f2aeda86528c80d0350c2bd0923ab41" +checksum = "8c358788aa585f51a78b11bec5d4b16fbe26dda1cc149f21d95dc24836a0be83" dependencies = [ "futures-util", - "http 0.2.12", - "hyper 0.14.28", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-util", "jsonrpsee-core", "jsonrpsee-types", "pin-project", @@ -3190,12 +3340,11 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "150d6168405890a7a3231a3c74843f58b8959471f6df76078db2619ddee1d07d" +checksum = "0feba38a9878d70ccccd2f54b534b15e861d6caa7911d59abfd3e0d8b4de091f" dependencies = [ - "anyhow", - "beef", + "http 1.1.0", "serde", "serde_json", "thiserror", @@ -3203,9 +3352,9 @@ dependencies = [ [[package]] name = "jsonrpsee-wasm-client" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f448d8eacd945cc17b6c0b42c361531ca36a962ee186342a97cdb8fca679cd77" +checksum = "e0da29b570ad645e72c6098716719d2ef58d51a8eb0f084ac5e43a5763839542" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -3214,11 +3363,11 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.22.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58b9db2dfd5bb1194b0ce921504df9ceae210a345bc2f6c5a61432089bbab070" +checksum = "e82b1c7fc629c345581d89ad802d53dc11d18df11d4d94d2c95da6e70f8520d3" dependencies = [ - "http 0.2.12", + "http 1.1.0", "jsonrpsee-client-transport", "jsonrpsee-core", "jsonrpsee-types", @@ -3294,15 +3443,6 @@ dependencies = [ "redb", ] -[[package]] -name = "kvq_store_rocksdb" -version = "0.1.0" -dependencies = [ - "anyhow", - "kvq", - "rocksdb", -] - [[package]] name = "lalrpop" version = "0.20.2" @@ -3389,22 +3529,6 @@ dependencies = [ "libc", ] -[[package]] -name = "librocksdb-sys" -version = "0.11.0+8.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" -dependencies = [ - "bindgen 0.65.1", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - [[package]] name = "libz-sys" version = "1.1.16" @@ -3439,16 +3563,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "lz4-sys" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "matchers" version = "0.1.0" @@ -3465,7 +3579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest 0.10.7", + "digest", ] [[package]] @@ -3489,6 +3603,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3747,12 +3871,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "open-fastrlp" version = "0.1.4" @@ -3906,7 +4024,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest 0.10.7", + "digest", "hmac", "password-hash", "sha2", @@ -3918,16 +4036,10 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest 0.10.7", + "digest", "hmac", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" version = "1.1.1" @@ -4133,7 +4245,7 @@ dependencies = [ "bincode", "clap", "curve25519-dalek", - "digest 0.10.7", + "digest", "dotenv", "ed25519-dalek", "env_logger 0.10.2", @@ -4522,7 +4634,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", - "hyper-rustls", + "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -4597,7 +4709,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -4622,17 +4734,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rocksdb" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" -dependencies = [ - "libc", - "librocksdb-sys", - "serde", -] - [[package]] name = "route-recognizer" version = "0.3.1" @@ -4642,7 +4743,7 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rsmq" version = "9.0.0" -source = "git+https://github.com/GopherJ/rsmq-async-rs?rev=72358a7c4b946ff01515e111ccbe8b0f8a04ede2#72358a7c4b946ff01515e111ccbe8b0f8a04ede2" +source = "git+https://github.com/GopherJ/rsmq-async-rs?rev=2af5a5c06a0341cb60f73ba0f0af066f65fb5e77#2af5a5c06a0341cb60f73ba0f0af066f65fb5e77" dependencies = [ "lazy_static", "r2d2", @@ -4664,6 +4765,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc-hex" version = "2.1.0" @@ -4706,30 +4813,19 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ "log", + "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework", -] - [[package]] name = "rustls-native-certs" version = "0.7.0" @@ -4768,6 +4864,33 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +[[package]] +name = "rustls-platform-verifier" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e3beb939bcd33c269f4bf946cc829fcd336370267c4a927ac0399c84a3151a1" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.11", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.102.5", + "security-framework", + "security-framework-sys", + "webpki-roots 0.26.1", + "winapi", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84e217e7fdc8466b5b35d30f8c0a30febd29173df4a3a0c2115d306b9c4117ad" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4780,9 +4903,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -4969,6 +5092,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "libc", + "num-bigint 0.4.5", "security-framework-sys", ] @@ -5143,19 +5267,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha1" version = "0.10.6" @@ -5164,7 +5275,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -5181,7 +5292,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -5203,7 +5314,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest 0.10.7", + "digest", "keccak", ] @@ -5271,7 +5382,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest 0.10.7", + "digest", "rand_core", ] @@ -5320,18 +5431,18 @@ dependencies = [ [[package]] name = "soketto" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +checksum = "37468c595637c10857701c990f93a40ce0e357cedb0953d1c26c8d8027f9bb53" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", "bytes", "futures", - "http 0.2.12", + "http 1.1.0", "httparse", "log", "rand", - "sha-1", + "sha1", ] [[package]] @@ -5714,11 +5825,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.22.4", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -5817,12 +5928,48 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "hdrhistogram", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bitflags 2.5.0", + "bytes", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "http-range-header", + "httpdate", + "iri-string", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", "tokio", + "tokio-util", + "tower", "tower-layer", "tower-service", "tracing", + "uuid 1.8.0", ] [[package]] @@ -5988,6 +6135,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -6088,6 +6244,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ + "getrandom", "serde", ] @@ -6531,7 +6688,7 @@ dependencies = [ "pbkdf2 0.11.0", "sha1", "time", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -6540,7 +6697,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe 7.2.0", ] [[package]] @@ -6553,6 +6719,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.10+zstd.1.5.6" diff --git a/Cargo.toml b/Cargo.toml index 3038a53e..93d9e3da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,17 +9,20 @@ members = [ "city_macros", "kvq", "kvq_store_redb", - "kvq_store_rocksdb", + # "kvq_store_rocksdb", "city_rollup_cli", "city_rollup_user_cli", "city_rollup_dev_cli", "city_rollup_core_node", "city_redis_store", "city_rollup_core_worker", + "city_rollup_core_worker_qbench", "city_rollup_worker_dispatch", "city_rollup_core_orchestrator", "city_rollup_core_api", - "city_rollup_rpc_provider" ] + "city_rollup_rpc_provider", + "city_rollup_user_prover_api" +] resolver = "2" [workspace.dependencies] @@ -91,6 +94,8 @@ tracing-log = "0.2.0" tracing-subscriber = { version = "0.3.17", features = ["std", "env-filter"] } shadow-rs = "0.27.1" +tower-http = { version = "0.5.2", features = ["full"] } +tower = { version = "0.4.13", features = ["full"] } plonky2 = { git = "https://github.com/QEDProtocol/plonky2-hwa", rev = "6a8ca008da97890b67a84f64784cfbc488b5238d" } @@ -110,13 +115,13 @@ env_logger = { version = "0.9.0", default-features = false } hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "serde"] } # NOTE: When upgrading, see `ahash` dependency. url = "2.5.0" redis = { version = "^0.25", default-features = false, features = ["acl", "keep-alive", "script"] } -rsmq = { git = "https://github.com/GopherJ/rsmq-async-rs", rev = "72358a7c4b946ff01515e111ccbe8b0f8a04ede2", default-features = false } +rsmq = { git = "https://github.com/GopherJ/rsmq-async-rs", rev = "2af5a5c06a0341cb60f73ba0f0af066f65fb5e77", default-features = false } r2d2 = { git = "https://github.com/GopherJ/r2d2", rev = "976ca74a74871d78764da2a040edcadad1af9be9" } r2d2_redis = { git = "https://github.com/GopherJ/r2d2-redis", rev = "63be37454cf441b832251aa915b1a9cb8bc27bff" } gnark-plonky2-wrapper = { git = "https://github.com/cf/gnark-plonky2-verifier", rev = "75681e2a0e20270734f082145d13cdfe3349d009" } bs58 = { version = "0.5.1", features = ["std", "check"] } -jsonrpsee = { version = "0.22.5", features = ["full"] } +jsonrpsee = { version = "0.24.0", features = ["full"] } rocksdb = { version = "0.21.0", features = ["serde", "multi-threaded-cf"] } home = { version = "0.5.9" } diff --git a/Makefile b/Makefile index 37cc11b3..7e776b2a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ TRACE_ENABLED := 1 PROFILE := release LOG_LEVEL := info,city_common_circuit=off,city_rollup_circuit=off,plonky2=off,city_crypto=off,city_store=off,city_rollup_common=off -DOCKER_PROFILE := lite .PHONY: check check: @@ -51,7 +50,7 @@ run-orchestrator: build-if-not-exists .PHONY: run-l2-worker run-l2-worker: build-if-not-exists - @RUST_LOG=${LOG_LEVEL} RUST_BACKTRACE=${TRACE_ENABLED} ./target/${PROFILE}/city-rollup-cli l2-worker + @RUST_LOG=${LOG_LEVEL} RUST_BACKTRACE=${TRACE_ENABLED} ./target/${PROFILE}/city-rollup-cli l2-worker --debug-mode 1 .PHONY: run-l2-worker-g16 run-l2-worker-g16: build-if-not-exists @@ -112,7 +111,6 @@ prove_sighash_0_hints: build-if-not-exists launch: @docker-compose \ -f docker-compose.yml \ - --profile ${DOCKER_PROFILE} \ up \ --build \ -d \ diff --git a/README.md b/README.md index 79089674..e69e7dca 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -

City Rollup

+

City Rollup

City Rollup

@@ -12,6 +12,7 @@ - docker - docker-compose - go +- jq - rust toolchain diff --git a/city_common/src/cli/args.rs b/city_common/src/cli/args.rs index 3e8d3ddb..679e5d99 100644 --- a/city_common/src/cli/args.rs +++ b/city_common/src/cli/args.rs @@ -1,6 +1,6 @@ use clap::Args; -use super::modes::QWorkerMode; +use super::modes::{QDumpInspectionData, QWorkerMode}; #[derive(Clone, Args)] pub struct RPCServerArgs { @@ -23,6 +23,10 @@ pub struct RPCServerArgs { pub struct OrchestratorArgs { #[clap(long, default_value = "0.0.0.0:7777", env)] pub server_addr: String, + + #[clap(long, default_value = "true", env)] + pub expose_proof_store_api: bool, + #[clap( env, long, @@ -63,6 +67,56 @@ pub struct L2WorkerArgs { #[clap(long, short, default_value = "0")] pub debug_mode: u32, } + + +#[derive(Clone, Args)] +pub struct L2DumpProofStoreArgs { + #[clap( + env, + long, + default_value = "redis://localhost:6379/0", + env + )] + pub redis_uri: String, + #[clap(short, long, default_value = "dogeregtest", env)] + pub network: String, + + #[clap(long, short)] + pub checkpoint_id: u64, + + #[clap(long, short)] + pub output: String, +} + +#[derive(Clone, Args)] +pub struct InspectL2DumpArgs { + #[clap(long, short)] + pub input: String, + + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] + pub data: Vec, + + #[clap(short, long, default_value = "")] + pub output: String, +} + + +#[derive(Clone, Args)] +pub struct QBenchArgs { + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] + pub input: Vec, + + #[clap(long, short)] + pub output: String, + + #[clap(short, long, default_value = "dogeregtest", env)] + pub network: String, + + #[clap(long, short, default_value = "1")] + pub num_iterations: u32, +} + + #[derive(Clone, Args)] pub struct L2TransferArgs { #[clap(long, default_value = "0.0.0.0:3000", env)] diff --git a/city_common/src/cli/modes.rs b/city_common/src/cli/modes.rs index fd2af302..5538afda 100644 --- a/city_common/src/cli/modes.rs +++ b/city_common/src/cli/modes.rs @@ -1,44 +1,53 @@ use clap::ValueEnum; use serde_repr::{Deserialize_repr, Serialize_repr}; - #[derive( - Serialize_repr, Deserialize_repr, PartialEq, Debug, Clone, Copy, Eq, Hash, PartialOrd, Ord, ValueEnum + Serialize_repr, + Deserialize_repr, + PartialEq, + Debug, + Clone, + Copy, + Eq, + Hash, + PartialOrd, + Ord, + ValueEnum, )] #[repr(u32)] pub enum QWorkerMode { - All = 0, - NoGroth16 = 1, - OnlyGroth16 = 2, + All = 0, + NoGroth16 = 1, + OnlyGroth16 = 2, } impl QWorkerMode { - pub fn to_u32(&self) -> u32 { - *self as u32 - } - pub fn is_groth16_enabled(&self) -> bool { - match self { - QWorkerMode::All => true, - QWorkerMode::NoGroth16 => false, - QWorkerMode::OnlyGroth16 => true, + pub fn to_u32(&self) -> u32 { + *self as u32 + } + pub fn is_groth16_enabled(&self) -> bool { + match self { + QWorkerMode::All => true, + QWorkerMode::NoGroth16 => false, + QWorkerMode::OnlyGroth16 => true, + } } - } } impl From for u32 { - fn from(value: QWorkerMode) -> u32 { - value as u32 - } + fn from(value: QWorkerMode) -> u32 { + value as u32 + } } impl TryFrom for QWorkerMode { - type Error = anyhow::Error; + type Error = anyhow::Error; - fn try_from(value: u32) -> Result { - match value { - 0 => Ok(QWorkerMode::All), - 1 => Ok(QWorkerMode::NoGroth16), - 2 => Ok(QWorkerMode::OnlyGroth16), - _ => Err(anyhow::format_err!("Invalid QWorkerMode value: {}", value)), - } - } + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(QWorkerMode::All), + 1 => Ok(QWorkerMode::NoGroth16), + 2 => Ok(QWorkerMode::OnlyGroth16), + _ => Err(anyhow::format_err!("Invalid QWorkerMode value: {}", value)), + } + } } impl ToString for QWorkerMode { @@ -49,4 +58,68 @@ impl ToString for QWorkerMode { QWorkerMode::OnlyGroth16 => "only-groth16".to_string(), } } -} \ No newline at end of file +} + +#[derive( + Serialize_repr, + Deserialize_repr, + PartialEq, + Debug, + Clone, + Copy, + Eq, + Hash, + PartialOrd, + Ord, + ValueEnum, +)] +#[repr(u32)] +pub enum QDumpInspectionData { + DependencyMap = 0, + JobConfig = 1, + SignatureProofIds = 2, + ProofWitnesses = 3, + ProofPublicInputs = 4, + Proofs = 5, +} +impl QDumpInspectionData { + pub fn to_u32(&self) -> u32 { + *self as u32 + } +} +impl From for u32 { + fn from(value: QDumpInspectionData) -> u32 { + value as u32 + } +} +impl TryFrom for QDumpInspectionData { + type Error = anyhow::Error; + + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(QDumpInspectionData::DependencyMap), + 1 => Ok(QDumpInspectionData::JobConfig), + 2 => Ok(QDumpInspectionData::SignatureProofIds), + 3 => Ok(QDumpInspectionData::ProofWitnesses), + 4 => Ok(QDumpInspectionData::ProofPublicInputs), + 5 => Ok(QDumpInspectionData::Proofs), + _ => Err(anyhow::format_err!( + "Invalid QDumpInspectionData value: {}", + value + )), + } + } +} + +impl ToString for QDumpInspectionData { + fn to_string(&self) -> String { + match *self { + QDumpInspectionData::DependencyMap => "dependency-map".to_string(), + QDumpInspectionData::JobConfig => "job-config".to_string(), + QDumpInspectionData::SignatureProofIds => "signature-proof-ids".to_string(), + QDumpInspectionData::ProofWitnesses => "proof-witnesses".to_string(), + QDumpInspectionData::ProofPublicInputs => "proof-public-inputs".to_string(), + QDumpInspectionData::Proofs => "proofs".to_string(), + } + } +} diff --git a/city_common/src/cli/user_args.rs b/city_common/src/cli/user_args.rs index ee75b042..61a6c4b5 100644 --- a/city_common/src/cli/user_args.rs +++ b/city_common/src/cli/user_args.rs @@ -21,6 +21,16 @@ pub struct RPCReplArgs { pub rpc_address: String, } + +#[derive(Clone, Args)] +pub struct ProverRPCArgs { + #[clap(long, short, default_value = "0.0.0.0:1447", env)] + pub prover_rpc_address: String, + #[clap(long, short, default_value = "")] + pub api_key: String, +} + + #[derive(Clone, Args)] pub struct GetPublicKeyArgs { /// user private key diff --git a/city_common/src/data/kv.rs b/city_common/src/data/kv.rs new file mode 100644 index 00000000..7b80855c --- /dev/null +++ b/city_common/src/data/kv.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + + +#[derive(Serialize, Deserialize, PartialEq, Clone)] +pub struct SimpleKVPair { + pub key: K, + pub value: V, +} diff --git a/city_common/src/data/mod.rs b/city_common/src/data/mod.rs index c769855c..573a1266 100644 --- a/city_common/src/data/mod.rs +++ b/city_common/src/data/mod.rs @@ -1,2 +1,3 @@ pub mod u8bytes; pub mod varuint; +pub mod kv; \ No newline at end of file diff --git a/city_common/src/data/u8bytes.rs b/city_common/src/data/u8bytes.rs index 7239873a..c8f51763 100644 --- a/city_common/src/data/u8bytes.rs +++ b/city_common/src/data/u8bytes.rs @@ -4,6 +4,15 @@ use serde::{Deserialize, Serialize}; use serde_with::serde_as; use std::fmt::Display; +#[serde_as] +#[derive(Serialize, Deserialize, PartialEq, Clone, Copy, Eq, PartialOrd, Ord, Debug, Hash)] +pub struct U8BytesFixed(#[serde_as(as = "serde_with::hex::Hex")] pub [u8; N]); + +impl Display for U8BytesFixed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex::encode(&self.0)) + } +} #[serde_as] #[derive(Serialize, Deserialize, PartialEq, Clone, Debug)] pub struct U8Bytes(#[serde_as(as = "serde_with::hex::Hex")] pub Vec); diff --git a/city_common_circuit/src/wallet/zk.rs b/city_common_circuit/src/wallet/zk.rs index 2b26176e..95f20aea 100644 --- a/city_common_circuit/src/wallet/zk.rs +++ b/city_common_circuit/src/wallet/zk.rs @@ -124,6 +124,16 @@ where hash_public_key_to_fingerprint: HashMap::new(), } } + pub fn get_fingerprint_public_key_for_private_key(&self, private_key: QHashOut) -> QHashOut{ + let pk = SimpleL2PrivateKey::new(private_key); + let hash_public_key = pk.get_public_key::(); + let fixed_circuit = ZKSignatureCircuitSimpleFixedPublicKey::new_from_isc( + &self.inner_circuit, + hash_public_key, + ); + fixed_circuit.get_fingerprint() + + } } impl + 'static, const D: usize> ZKSignatureBasicWalletProvider @@ -151,7 +161,20 @@ where anyhow::bail!("Key not found"); } } else { - anyhow::bail!("Key not found"); + let fixed_circuit = ZKSignatureCircuitSimpleFixedPublicKey::new_from_isc( + &self.inner_circuit, + hash_public_key, + ); + let circuit_fingerprint_public_key = fixed_circuit.get_fingerprint(); + let key = SimpleZKSignatureWalletKey { + circuit_fingerprint_public_key, + hash_public_key, + fixed_circuit, + }; + let inner_proof = self.inner_circuit.prove_base(private_key, action_hash)?; + let fixed_proof = key.fixed_circuit.prove_base(&inner_proof)?; + self.wrapper_circuit + .prove_base(&fixed_proof, key.fixed_circuit.get_verifier_config_ref()) } } diff --git a/city_crypto/src/hash/base_types/hash256.rs b/city_crypto/src/hash/base_types/hash256.rs index c76ffd30..a3977e30 100644 --- a/city_crypto/src/hash/base_types/hash256.rs +++ b/city_crypto/src/hash/base_types/hash256.rs @@ -1,4 +1,4 @@ -use std::fmt::Display; +use std::{fmt::Display, ops}; use hex::FromHexError; use kvq::traits::KVQSerializable; @@ -24,7 +24,17 @@ impl Default for Hash256 { } } +impl ops::BitXor for Hash256 { + type Output = Hash256; + + fn bitxor(self, rhs: Hash256) -> Hash256 { + Hash256(core::array::from_fn(|i| self.0[i] ^ rhs.0[i])) + } +} + + impl Hash256 { + pub const ZERO: Self = Self([0u8; 32]); pub fn from_hex_string(s: &str) -> Result { let bytes = hex::decode(s)?; assert_eq!(bytes.len(), 32); @@ -46,6 +56,52 @@ impl Hash256 { pub fn reversed(&self) -> Self { Hash256(core::array::from_fn(|i| self.0[31 - i])) } + pub fn to_le_u64_x4(&self) -> [u64; 4] { + + [ + u64::from_le_bytes([ + self.0[0], + self.0[1], + self.0[2], + self.0[3], + self.0[4], + self.0[5], + self.0[6], + self.0[7], + ]), + u64::from_le_bytes([ + self.0[8], + self.0[9], + self.0[10], + self.0[11], + self.0[12], + self.0[13], + self.0[14], + self.0[15], + ]), + u64::from_le_bytes([ + self.0[16], + self.0[17], + self.0[18], + self.0[19], + self.0[20], + self.0[21], + self.0[22], + self.0[23], + ]), + u64::from_le_bytes([ + self.0[24], + self.0[25], + self.0[26], + self.0[27], + self.0[28], + self.0[29], + self.0[30], + self.0[31], + ]), + + ] + } } impl From for ECDSASecretKey { diff --git a/city_crypto/src/hash/merkle/treeprover/mod.rs b/city_crypto/src/hash/merkle/treeprover/mod.rs index ebc78868..94df6918 100644 --- a/city_crypto/src/hash/merkle/treeprover/mod.rs +++ b/city_crypto/src/hash/merkle/treeprover/mod.rs @@ -237,6 +237,7 @@ pub trait TPLeafAggregator { fn get_output_from_right_leaf(left: &IO, right: &IL) -> IO; fn get_output_from_leaves(left: &IL, right: &IL) -> IO; } + pub struct AggWTLeafAggregator; impl, F: RichField> TPLeafAggregator> diff --git a/city_crypto/src/hash/qhashout.rs b/city_crypto/src/hash/qhashout.rs index e70aebbe..0c0e5d3b 100644 --- a/city_crypto/src/hash/qhashout.rs +++ b/city_crypto/src/hash/qhashout.rs @@ -172,6 +172,17 @@ impl QHashOut { pub fn to_felt248_hash256(&self) -> Hash256 { felt248_hashout_to_hash256_le(self.0) } + pub fn from_hash256_le(hash: Hash256) -> Self { + let u64_x4 = hash.to_le_u64_x4(); + Self(HashOut { + elements: [ + F::from_noncanonical_u64(u64_x4[0]), + F::from_noncanonical_u64(u64_x4[1]), + F::from_noncanonical_u64(u64_x4[2]), + F::from_noncanonical_u64(u64_x4[3]), + ], + }) + } pub fn to_string_le(&self) -> String { hex::encode(self.to_le_bytes()) } diff --git a/city_crypto/src/signature/secp256k1/core.rs b/city_crypto/src/signature/secp256k1/core.rs index a14cd502..c5f91bf5 100644 --- a/city_crypto/src/signature/secp256k1/core.rs +++ b/city_crypto/src/signature/secp256k1/core.rs @@ -48,7 +48,6 @@ pub fn secp256k1_base_from_bytes(bytes: &[u8], offset: usize) -> Secp256K1Base { }); Secp256K1Base(arr) } - #[serde_as] #[derive(Serialize, Deserialize, PartialEq, Clone, Copy, Debug)] pub struct QEDCompressedSecp256K1Signature { diff --git a/city_redis_store/src/lib.rs b/city_redis_store/src/lib.rs index c5b91be3..89e9721d 100644 --- a/city_redis_store/src/lib.rs +++ b/city_redis_store/src/lib.rs @@ -93,4 +93,20 @@ impl QProofStoreWriterSync for RedisStore { conn.hset_nx(PROOFS, <[u8; 24]>::from(&id).to_vec(), data)?; Ok(()) } + + fn write_next_jobs( + &mut self, + jobs: &[QProvingJobDataID], + next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + self.write_next_jobs_core(jobs, next_jobs) + } + + fn write_multidimensional_jobs( + &mut self, + jobs_levels: &[Vec], + next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + self.write_multidimensional_jobs_core(jobs_levels, next_jobs) + } } diff --git a/city_rollup_circuit/src/lib.rs b/city_rollup_circuit/src/lib.rs index cee3bb3d..38d3186a 100644 --- a/city_rollup_circuit/src/lib.rs +++ b/city_rollup_circuit/src/lib.rs @@ -6,3 +6,4 @@ pub mod common_gates; pub mod sighash_circuits; pub mod verify_templates; pub mod worker; +pub mod wallet; \ No newline at end of file diff --git a/city_rollup_circuit/src/wallet/memory.rs b/city_rollup_circuit/src/wallet/memory.rs new file mode 100644 index 00000000..2613dd4f --- /dev/null +++ b/city_rollup_circuit/src/wallet/memory.rs @@ -0,0 +1,221 @@ +use city_common::{ + binaryhelpers::bytes::CompressedPublicKey, + config::rollup_constants::{DEPOSIT_FEE_AMOUNT, WITHDRAWAL_FEE_AMOUNT}, +}; +use city_common_circuit::{ + circuits::l1_secp256k1_signature::L1Secp256K1SignatureCircuit, + wallet::zk::{MemoryZKSignatureWallet, SimpleZKSignatureWallet, ZKSignatureBasicWalletProvider, ZKSignatureWalletProvider}, +}; +use city_crypto::{ + hash::{ + base_types::{hash160::Hash160, hash256::Hash256}, + qhashout::QHashOut, + }, + signature::secp256k1::{ + core::QEDCompressedSecp256K1Signature, + wallet::{MemorySecp256K1Wallet, Secp256K1WalletProvider}, + }, +}; +use city_rollup_common::{ + api::data::{ + block::rpc_request::{ + CityAddWithdrawalRPCRequest, CityClaimDepositRPCRequest, CityTokenTransferRPCRequest, + }, + store::CityL1Deposit, + }, + introspection::rollup::signature::QEDSigAction, +}; +use plonky2::{ + hash::poseidon::PoseidonHash, + plonk::{ + config::{AlgebraicHasher, GenericConfig}, + proof::ProofWithPublicInputs, + }, +}; + +#[derive(Clone)] +pub struct CityMemoryWallet + 'static, const D: usize> +where + C::Hasher: AlgebraicHasher, +{ + pub zk_wallet: MemoryZKSignatureWallet>, + pub secp256k1_circuit: Option>, + pub secp256k1_wallet: MemorySecp256K1Wallet, +} + +impl + 'static, const D: usize> CityMemoryWallet +where + C::Hasher: AlgebraicHasher, +{ + pub fn new() -> Self { + Self { + zk_wallet: MemoryZKSignatureWallet::>::new_memory(), + secp256k1_wallet: MemorySecp256K1Wallet::new(), + secp256k1_circuit: Some(L1Secp256K1SignatureCircuit::new()), + } + } + pub fn new_fast_setup() -> Self { + Self { + zk_wallet: MemoryZKSignatureWallet::>::new_memory(), + secp256k1_wallet: MemorySecp256K1Wallet::new(), + secp256k1_circuit: None, + } + } + pub fn setup_circuits(&mut self) { + if self.secp256k1_circuit.is_none() { + self.secp256k1_circuit = Some(L1Secp256K1SignatureCircuit::new()); + } + } + pub fn add_zk_private_key(&mut self, private_key: QHashOut) -> QHashOut { + self.zk_wallet.add_private_key(private_key) + } + pub fn add_secp256k1_private_key( + &mut self, + private_key: Hash256, + ) -> anyhow::Result { + self.secp256k1_wallet.add_private_key(private_key) + } + pub fn zk_sign( + &self, + fingerprint_hash: QHashOut, + message: QHashOut, + ) -> anyhow::Result> { + self.zk_wallet.sign(fingerprint_hash, message) + } + pub fn zk_sign_with_private_key( + &self, + private_key: QHashOut, + action_hash: QHashOut, + ) -> anyhow::Result> { + self.zk_wallet.zk_sign(private_key, action_hash) + } + pub fn zk_sign_hash_public_key( + &self, + hash_public_key: QHashOut, + message: QHashOut, + ) -> anyhow::Result> { + self.zk_wallet + .sign_by_hash_public_key(hash_public_key, message) + } + pub fn sign_secp256k1( + &self, + public_key: CompressedPublicKey, + message: Hash256, + ) -> anyhow::Result { + self.secp256k1_wallet.sign(&public_key, message) + } + pub fn sign_hash_secp256k1( + &self, + public_key: CompressedPublicKey, + message: QHashOut, + ) -> anyhow::Result { + self.secp256k1_wallet.sign_qhashout(&public_key, message) + } + pub fn zk_secp256k1_from_signature( + &self, + signature: &QEDCompressedSecp256K1Signature, + ) -> anyhow::Result> { + self.secp256k1_circuit.as_ref().unwrap().prove(signature) + } + pub fn zk_sign_secp256k1( + &self, + public_key: CompressedPublicKey, + message: Hash256, + ) -> anyhow::Result> { + let ecc_sig = self.sign_secp256k1(public_key, message)?; + self.secp256k1_circuit.as_ref().unwrap().prove(&ecc_sig) + } + pub fn zk_sign_hash_secp256k1( + &self, + public_key: CompressedPublicKey, + message: QHashOut, + ) -> anyhow::Result> { + let ecc_sig = self.sign_hash_secp256k1(public_key, message)?; + self.secp256k1_circuit.as_ref().unwrap().prove(&ecc_sig) + } + pub fn sign_claim_deposit( + &self, + network_magic: u64, + user_id: u64, + l1_deposit: &CityL1Deposit, + ) -> anyhow::Result { + let sig_preimage = QEDSigAction::::new_claim_deposit_action( + network_magic, + user_id, + l1_deposit.txid, + l1_deposit.value, + DEPOSIT_FEE_AMOUNT, + ); + tracing::info!( + "sig_preimage: {}", + serde_json::to_string(&sig_preimage).unwrap() + ); + let hash = sig_preimage.get_qhash::(); + let proof = self.zk_sign_hash_secp256k1(l1_deposit.public_key, hash)?; + + let signature_proof = bincode::serialize(&proof)?; + Ok(CityClaimDepositRPCRequest { + public_key: l1_deposit.public_key.0, + user_id, + deposit_id: l1_deposit.deposit_id, + value: l1_deposit.value, + txid: l1_deposit.txid, + signature_proof, + }) + } + pub fn sign_l2_transfer( + &self, + public_key: QHashOut, + network_magic: u64, + from: u64, + to: u64, + value: u64, + nonce: u64, + ) -> anyhow::Result { + let sig_preimage = + QEDSigAction::::new_transfer_action(network_magic, from, nonce, to, value); + let hash = sig_preimage.get_qhash::(); + let proof = self.zk_sign(public_key, hash)?; + + let signature_proof = bincode::serialize(&proof)?; + Ok(CityTokenTransferRPCRequest { + signature_proof, + to, + nonce, + user_id: from, + value: value, + }) + } + pub fn sign_withdrawal( + &self, + public_key: QHashOut, + network_magic: u64, + user_id: u64, + l1_address: Hash160, + value: u64, + nonce: u64, + ) -> anyhow::Result { + let sig_preimage: QEDSigAction = + QEDSigAction::::new_withdrawal_action::( + network_magic, + user_id, + nonce, + l1_address, + 0, + value, + WITHDRAWAL_FEE_AMOUNT, + ); + let hash = sig_preimage.get_qhash::(); + let proof = self.zk_sign(public_key, hash)?; + + let signature_proof = bincode::serialize(&proof)?; + Ok(CityAddWithdrawalRPCRequest { + signature_proof, + nonce, + user_id: user_id, + value: value, + destination_type: 0, + destination: l1_address, + }) + } +} diff --git a/city_rollup_circuit/src/wallet/mod.rs b/city_rollup_circuit/src/wallet/mod.rs new file mode 100644 index 00000000..22aba366 --- /dev/null +++ b/city_rollup_circuit/src/wallet/mod.rs @@ -0,0 +1 @@ +pub mod memory; \ No newline at end of file diff --git a/city_rollup_circuit/src/worker/bench.rs b/city_rollup_circuit/src/worker/bench.rs new file mode 100644 index 00000000..f2932302 --- /dev/null +++ b/city_rollup_circuit/src/worker/bench.rs @@ -0,0 +1,79 @@ +use std::{marker::PhantomData, time::Instant}; + +use city_crypto::hash::{merkle::treeprover::TPCircuitFingerprintConfig, qhashout::QHashOut}; +use city_rollup_common::{block_template::data::CityGroth16ProofData, qworker::{job_id::{ProvingJobCircuitType, QProvingJobDataID, QWorkerJobBenchmark}, proof_store::QProofStoreReaderSync, verifier::QWorkerVerifyHelper}}; +use plonky2::plonk::{circuit_data::{CommonCircuitData, VerifierOnlyCircuitData}, config::GenericConfig, proof::ProofWithPublicInputs}; + +use super::traits::{QWorkerGenericProverGroth16, QWorkerGenericProverMut}; + +pub struct QWorkerGenericProverMutBench, S: QProofStoreReaderSync, C: GenericConfig, const D: usize> { + pub prover: P, + pub benchmarks: Vec, + _store: PhantomData, + _config: PhantomData, +} + +impl, S: QProofStoreReaderSync, C: GenericConfig, const D: usize> QWorkerGenericProverMutBench { + pub fn new(prover: P) -> Self { + Self { + prover, + benchmarks: Vec::new(), + _store: PhantomData, + _config: PhantomData, + } + } + pub fn add_benchmark(&mut self, job_id: QProvingJobDataID, duration: u64) { + self.benchmarks.push(QWorkerJobBenchmark { + job_id: job_id.to_fixed_bytes(), + duration, + }); + } + pub fn get_benchmarks(&self) -> &[QWorkerJobBenchmark] { + &self.benchmarks + } + +} +impl , S: QProofStoreReaderSync, C: GenericConfig, const D: usize> QWorkerVerifyHelper for QWorkerGenericProverMutBench { + fn get_tree_prover_fingerprint_config( + &self, + circuit_type: ProvingJobCircuitType, + ) -> anyhow::Result> { + self.prover.get_tree_prover_fingerprint_config(circuit_type) + } + + fn get_verifier_triplet_for_circuit_type( + &self, + circuit_type: ProvingJobCircuitType, + ) -> ( + &CommonCircuitData, + &VerifierOnlyCircuitData, + QHashOut, + ) { + self.prover.get_verifier_triplet_for_circuit_type(circuit_type) + } +} +impl, S: QProofStoreReaderSync, C: GenericConfig, const D: usize> QWorkerGenericProverMut for QWorkerGenericProverMutBench { + fn worker_prove_mut( + &mut self, + store: &S, + job_id: QProvingJobDataID, + ) -> anyhow::Result> { + let start_time = Instant::now(); + + let result = self.prover.worker_prove_mut(store, job_id); + if result.is_ok() { + let duration = start_time.elapsed().as_millis() as u64; + self.add_benchmark(job_id, duration); + } + result + } +} +impl + QWorkerGenericProverGroth16, S: QProofStoreReaderSync, C: GenericConfig, const D: usize> QWorkerGenericProverGroth16 for QWorkerGenericProverMutBench { + fn worker_prove_groth16( + &self, + store: &S, + job_id: QProvingJobDataID, + ) -> anyhow::Result { + self.prover.worker_prove_groth16(store, job_id) + } +} \ No newline at end of file diff --git a/city_rollup_circuit/src/worker/mod.rs b/city_rollup_circuit/src/worker/mod.rs index b81fb9be..3140c7fb 100644 --- a/city_rollup_circuit/src/worker/mod.rs +++ b/city_rollup_circuit/src/worker/mod.rs @@ -2,3 +2,4 @@ pub mod prove_store; pub mod prover; pub mod toolbox; pub mod traits; +pub mod bench; \ No newline at end of file diff --git a/city_rollup_circuit/src/worker/traits.rs b/city_rollup_circuit/src/worker/traits.rs index c7528cb1..e749e0de 100644 --- a/city_rollup_circuit/src/worker/traits.rs +++ b/city_rollup_circuit/src/worker/traits.rs @@ -31,6 +31,7 @@ pub trait QWorkerGenericProverGroth16 anyhow::Result; } + pub trait QWorkerGenericProverMut, const D: usize>: QWorkerVerifyHelper { diff --git a/city_rollup_cli/Cargo.toml b/city_rollup_cli/Cargo.toml index feaa0adf..2f4b68d3 100644 --- a/city_rollup_cli/Cargo.toml +++ b/city_rollup_cli/Cargo.toml @@ -21,6 +21,7 @@ city_common = { path = "../city_common" } city_rollup_common = { path = "../city_rollup_common" } city_rollup_core_node = { path = "../city_rollup_core_node" } city_rollup_core_worker = { path = "../city_rollup_core_worker" } +city_rollup_core_worker_qbench = { path = "../city_rollup_core_worker_qbench" } city_rollup_core_api = { path = "../city_rollup_core_api" } city_rollup_core_orchestrator = { path = "../city_rollup_core_orchestrator" } bitcoincore-rpc = { workspace = true } diff --git a/city_rollup_cli/src/main.rs b/city_rollup_cli/src/main.rs index 04f3ac5c..c838b047 100644 --- a/city_rollup_cli/src/main.rs +++ b/city_rollup_cli/src/main.rs @@ -10,6 +10,9 @@ use crate::subcommand::apiserver; use crate::subcommand::l2worker; use crate::subcommand::orchestrator; use crate::subcommand::rpcserver; +use crate::subcommand::dumpblock; +use crate::subcommand::qbench; +use crate::subcommand::inspectdump; use crate::subcommand::Cli; use crate::subcommand::Commands; @@ -31,6 +34,15 @@ fn main() -> anyhow::Result<()> { Commands::APIServer(args) => { apiserver::run(args)?; } + Commands::DumpBlock(args) => { + dumpblock::run(args)?; + } + Commands::QBench(args) => { + qbench::run(args)?; + } + Commands::InspectDump(args) => { + inspectdump::run(args)?; + } }; Ok::<_, anyhow::Error>(()) } diff --git a/city_rollup_cli/src/subcommand.rs b/city_rollup_cli/src/subcommand.rs index 35c04052..b8df02b0 100644 --- a/city_rollup_cli/src/subcommand.rs +++ b/city_rollup_cli/src/subcommand.rs @@ -6,7 +6,9 @@ pub mod l2worker; pub mod orchestrator; pub mod rpcserver; pub mod apiserver; - +pub mod dumpblock; +pub mod qbench; +pub mod inspectdump; #[derive(Parser)] pub struct Cli { #[command(subcommand)] @@ -19,4 +21,7 @@ pub enum Commands { L2Worker(city_common::cli::args::L2WorkerArgs), Orchestrator(city_common::cli::args::OrchestratorArgs), APIServer(city_common::cli::args::APIServerArgs), + DumpBlock(city_common::cli::args::L2DumpProofStoreArgs), + QBench(city_common::cli::args::QBenchArgs), + InspectDump(city_common::cli::args::InspectL2DumpArgs), } diff --git a/city_rollup_cli/src/subcommand/dumpblock.rs b/city_rollup_cli/src/subcommand/dumpblock.rs new file mode 100644 index 00000000..491658ce --- /dev/null +++ b/city_rollup_cli/src/subcommand/dumpblock.rs @@ -0,0 +1,8 @@ +use city_common::cli::args::L2DumpProofStoreArgs; + + +#[tokio::main] +pub async fn run(args: L2DumpProofStoreArgs) -> anyhow::Result<()> { + city_rollup_core_worker_qbench::dump::run_dump_block_proof_store(&args)?; + Ok(()) +} diff --git a/city_rollup_cli/src/subcommand/inspectdump.rs b/city_rollup_cli/src/subcommand/inspectdump.rs new file mode 100644 index 00000000..e5a516f3 --- /dev/null +++ b/city_rollup_cli/src/subcommand/inspectdump.rs @@ -0,0 +1,8 @@ +use city_common::cli::args::InspectL2DumpArgs; + + +#[tokio::main] +pub async fn run(args: InspectL2DumpArgs) -> anyhow::Result<()> { + city_rollup_core_worker_qbench::inspect::run_inspect_l2_dump(&args)?; + Ok(()) +} diff --git a/city_rollup_cli/src/subcommand/qbench.rs b/city_rollup_cli/src/subcommand/qbench.rs new file mode 100644 index 00000000..bfc81e62 --- /dev/null +++ b/city_rollup_cli/src/subcommand/qbench.rs @@ -0,0 +1,8 @@ +use city_common::cli::args::QBenchArgs; + + +#[tokio::main] +pub async fn run(args: QBenchArgs) -> anyhow::Result<()> { + city_rollup_core_worker_qbench::qbench::run_qbench(&args)?; + Ok(()) +} diff --git a/city_rollup_common/src/actors/rpc_processor.rs b/city_rollup_common/src/actors/rpc_processor.rs index 4221180c..78a7ed59 100644 --- a/city_rollup_common/src/actors/rpc_processor.rs +++ b/city_rollup_common/src/actors/rpc_processor.rs @@ -90,6 +90,7 @@ impl OrchestratorEventSenderSync for CityScenarioRequestedActio } } +#[derive(Clone)] pub struct QRPCProcessor { pub checkpoint_id: u64, pub output: CityScenarioRequestedActionsFromRPC, diff --git a/city_rollup_common/src/actors/simple/events.rs b/city_rollup_common/src/actors/simple/events.rs index e4632489..23f9b007 100644 --- a/city_rollup_common/src/actors/simple/events.rs +++ b/city_rollup_common/src/actors/simple/events.rs @@ -2,17 +2,24 @@ use std::collections::VecDeque; use crate::{ actors::traits::{WorkerEventReceiverSync, WorkerEventTransmitterSync}, - qworker::job_id::QProvingJobDataID, + qworker::job_id::{QProvingJobDataID, QWorkerJobBenchmark}, }; pub struct CityEventProcessorMemory { pub job_queue: VecDeque, + pub benchmarks_enabled: bool, + pub benchmarks: Vec, pub core_job_completed: bool, } impl CityEventProcessorMemory { pub fn new() -> Self { + Self::new_with_config(false) + } + pub fn new_with_config(benchmarks_enabled: bool) -> Self { Self { job_queue: VecDeque::new(), + benchmarks_enabled, + benchmarks: Vec::new(), core_job_completed: true, } } @@ -36,6 +43,16 @@ impl WorkerEventReceiverSync for CityEventProcessorMemory { self.core_job_completed = true; Ok(()) } + + fn record_job_bench(&mut self, job: QProvingJobDataID, duration: u64) -> anyhow::Result<()> { + if self.benchmarks_enabled { + self.benchmarks.push(QWorkerJobBenchmark { + job_id: job.to_fixed_bytes(), + duration, + }); + } + Ok(()) + } } impl WorkerEventTransmitterSync for CityEventProcessorMemory { diff --git a/city_rollup_common/src/actors/traits.rs b/city_rollup_common/src/actors/traits.rs index 0f19bebe..97135061 100644 --- a/city_rollup_common/src/actors/traits.rs +++ b/city_rollup_common/src/actors/traits.rs @@ -108,6 +108,7 @@ pub trait WorkerEventReceiverSync { fn wait_for_next_job(&mut self) -> anyhow::Result; fn enqueue_jobs(&mut self, jobs: &[QProvingJobDataID]) -> anyhow::Result<()>; fn notify_core_goal_completed(&mut self, job: QProvingJobDataID) -> anyhow::Result<()>; + fn record_job_bench(&mut self, job: QProvingJobDataID, duration: u64) -> anyhow::Result<()>; } pub trait WorkerEventTransmitterSync { diff --git a/city_rollup_common/src/api/data/store/mod.rs b/city_rollup_common/src/api/data/store/mod.rs index a9e31b77..5d8a7db3 100644 --- a/city_rollup_common/src/api/data/store/mod.rs +++ b/city_rollup_common/src/api/data/store/mod.rs @@ -1,4 +1,4 @@ -use city_common::binaryhelpers::bytes::CompressedPublicKey; +use city_common::{binaryhelpers::bytes::CompressedPublicKey, data::u8bytes::U8BytesFixed}; use city_crypto::hash::{ base_types::{hash160::Hash160, hash256::Hash256}, qhashout::QHashOut, @@ -126,6 +126,38 @@ impl CityUserState { self.balance >= amount && self.nonce < nonce } } + +#[derive(Debug, Clone, Serialize, Deserialize, Copy, Hash, Eq, PartialEq)] +pub struct CityL1DepositJSON { + pub deposit_id: u64, + pub checkpoint_id: u64, + pub value: u64, + pub txid: Hash256, + pub public_key: U8BytesFixed<33>, +} +impl CityL1DepositJSON { + pub fn to_city_l1_deposit(&self) -> CityL1Deposit { + CityL1Deposit { + deposit_id: self.deposit_id, + checkpoint_id: self.checkpoint_id, + value: self.value, + txid: self.txid.reversed(), + public_key: CompressedPublicKey(self.public_key.0), + } + } + pub fn from_city_l1_deposit(deposit: &CityL1Deposit) -> Self { + Self { + deposit_id: deposit.deposit_id, + checkpoint_id: deposit.checkpoint_id, + value: deposit.value, + txid: deposit.txid.reversed(), + public_key: U8BytesFixed(deposit.public_key.0), + } + } + +} + + #[derive(Debug, Clone, Serialize, Deserialize, Copy, Hash, Eq, PartialEq)] pub struct CityL1Deposit { pub deposit_id: u64, @@ -134,6 +166,12 @@ pub struct CityL1Deposit { pub txid: Hash256, pub public_key: CompressedPublicKey, } +impl CityL1Deposit { + + pub fn to_json_variant(&self) -> CityL1DepositJSON { + CityL1DepositJSON::from_city_l1_deposit(&self) + } +} impl KVQSerializable for CityL1Deposit { fn to_bytes(&self) -> anyhow::Result> { // 8 + 8 + 8 + 32 + 33 = 89 bytes diff --git a/city_rollup_common/src/qworker/dump.rs b/city_rollup_common/src/qworker/dump.rs new file mode 100644 index 00000000..b2caa833 --- /dev/null +++ b/city_rollup_common/src/qworker/dump.rs @@ -0,0 +1,141 @@ +use std::collections::HashMap; + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_with::serde_as; + +use super::{job_id::{QProvingJobDataID, QProvingJobDataIDSerialized}, proof_store::QProofStoreReaderSync}; + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Eq, Hash, PartialOrd, Ord)] +pub struct QJobWithDependencies { + pub id: QProvingJobDataID, + pub dependencies: Vec, +} +impl QJobWithDependencies { + pub fn to_serialized(&self) -> QJobWithDependenciesSerialized { + self.into() + } + pub fn to_job_id_list(&self) -> Vec { + let mut result = vec![self.id]; + for dep in self.dependencies.iter() { + result.extend(dep.to_job_id_list()); + } + result + } + pub fn get_all_dependencies(&self) -> Vec { + let mut result = vec![]; + for dep in self.dependencies.iter() { + result.extend(dep.to_job_id_list()); + } + result + } +} +#[serde_as] +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Eq, Hash, PartialOrd, Ord)] +pub struct QJobWithDependenciesSerialized { + #[serde_as(as = "serde_with::hex::Hex")] + pub id: QProvingJobDataIDSerialized, + pub dependencies: Vec, +} +impl From<&QJobWithDependencies> for QJobWithDependenciesSerialized { + fn from(value: &QJobWithDependencies) -> Self { + Self { + id: value.id.to_fixed_bytes(), + dependencies: value.dependencies.iter().map(|x|x.into()).collect(), + } + } +} +#[derive(Debug, Clone)] +pub struct QDependencyMap { + pub dependencies: HashMap>, +} +impl QDependencyMap { + pub fn new() -> Self { + Self { + dependencies: HashMap::new(), + } + } + pub fn print_all(&self) { + for (key, value) in self.dependencies.iter() { + println!("{} ({:?}) -> {:?}", key.to_hex_string(), key, value); + } + } + pub fn add_dependency(&mut self, parent: QProvingJobDataID, dependency: QProvingJobDataID) { + let base = self.dependencies.get_mut(&parent); + if base.is_some() { + let base = base.unwrap(); + if !base.contains(&dependency) { + base.push(dependency) + } + }else{ + self.dependencies.insert(parent, vec![dependency]); + } + } + pub fn add_dependencies(&mut self, parent: QProvingJobDataID, dependencies: &[QProvingJobDataID]) { + let base = self.dependencies.get_mut(&parent); + if base.is_some() { + let base = base.unwrap(); + dependencies.into_iter().for_each(|dependency| { + if !base.contains(&dependency) { + base.push(*dependency) + } + }) + }else{ + self.dependencies.insert(parent, dependencies.to_vec()); + } + } + pub fn add_dependencies_multidimensional(&mut self, parents: &[QProvingJobDataID], dependencies: &[QProvingJobDataID]) { + parents.into_iter().for_each(|parent| { + self.add_dependencies(*parent, dependencies) + }) + } + pub fn get_dependencies(&self, id: QProvingJobDataID) -> Vec { + let result = self.dependencies.get(&id); + if result.is_some() { + result.unwrap().to_vec() + }else{ + vec![] + } + } + pub fn get_dependency_tree_for_block(&self, checkpoint_id: u64) -> QJobWithDependencies { + self.get_dependency_tree(QProvingJobDataID::notify_block_complete(checkpoint_id)) + } + pub fn get_dependency_tree(&self, root: QProvingJobDataID) -> QJobWithDependencies { + let dependencies = self.get_dependencies(root).into_iter().map(|id| { + self.get_dependency_tree(id) + }).collect::>(); + + QJobWithDependencies { + id: root, + dependencies + } + } + pub fn injest_leaf_jobs_from_store(&mut self, store: &PS, leaf_jobs: &[QProvingJobDataID]) -> anyhow::Result<()> { + if leaf_jobs.len() == 1 && leaf_jobs[0].is_notify_orchestrator_complete() { + Ok(()) + }else{ + let groups = leaf_jobs.iter().group_by(|x|x.get_sub_group_counter_goal_next_jobs_id()).into_iter().map(|(goal, group)|{ + let g = group.into_iter().map(|x| *x).collect::>(); + + (goal, g) + }).collect::>(); + + for (goal, dependencies) in groups.iter() { + let goal_next_counter = store.get_goal_by_job_id(dependencies[0])?; + if goal_next_counter != 0 { + let goal_next_jobs = store.get_next_jobs_by_job_id(*goal)?; + self.add_dependencies_multidimensional(&goal_next_jobs, dependencies); + self.injest_leaf_jobs_from_store(store, &goal_next_jobs)?; + } + } + + Ok(()) + } + } +} + +pub fn dump_job_dependencies_from_store(store: &PS, leaf_jobs: &[QProvingJobDataID]) -> anyhow::Result { + let mut dependency_map = QDependencyMap::new(); + dependency_map.injest_leaf_jobs_from_store(store, leaf_jobs)?; + Ok(dependency_map) +} \ No newline at end of file diff --git a/city_rollup_common/src/qworker/job_id.rs b/city_rollup_common/src/qworker/job_id.rs index e4154188..4243c4bf 100644 --- a/city_rollup_common/src/qworker/job_id.rs +++ b/city_rollup_common/src/qworker/job_id.rs @@ -1,6 +1,8 @@ use city_common::cli::modes::QWorkerMode; +use hex::FromHexError; use serde::{Deserialize, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; +use serde_with::serde_as; #[derive( Serialize_repr, Deserialize_repr, PartialEq, Debug, Clone, Copy, Eq, Hash, PartialOrd, Ord, @@ -175,6 +177,31 @@ impl From for u8 { pub type QProvingJobDataIDSerialized = [u8; 24]; +#[serde_as] +#[derive(Serialize, Deserialize, PartialEq, Copy, Eq, Hash, Clone, Debug)] +pub struct QProvingJobDataIDSerializedWrapped(#[serde_as(as = "serde_with::hex::Hex")] pub QProvingJobDataIDSerialized); + +impl QProvingJobDataIDSerializedWrapped { + pub fn from_hex_string(s: &str) -> Result { + let bytes = hex::decode(s)?; + assert_eq!(bytes.len(), 24); + let mut array = [0u8; 24]; + array.copy_from_slice(&bytes); + Ok(Self(array)) + } +} + +#[serde_as] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct QWorkerJobBenchmark { + #[serde_as(as = "serde_with::hex::Hex")] + pub job_id: QProvingJobDataIDSerialized, + + + pub duration: u64, +} + + #[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Copy, Eq, Hash, PartialOrd, Ord)] pub struct QProvingJobDataID { pub topic: QJobTopic, @@ -246,6 +273,18 @@ impl QProvingJobDataID { data_index, } } + pub fn core_op_witness(circuit_type: ProvingJobCircuitType, checkpoint_id: u64, task_index: u32) -> Self { + Self::new( + QJobTopic::GenerateStandardProof, + checkpoint_id, + circuit_type.to_circuit_group_id(), + 0, + task_index, + circuit_type, + ProvingJobDataType::InputWitness, + 0, + ) + } pub fn transfer_signature_proof(rpc_node_id: u32, block_id: u64, transfer_id: u32) -> Self { Self { topic: QJobTopic::BlockUserSignatureProof, @@ -429,6 +468,9 @@ impl QProvingJobDataID { ..*self } } + pub fn is_notify_orchestrator_complete(&self) -> bool { + self.topic == QJobTopic::NotifyOrchestratorComplete + } pub fn get_tree_parent_proof_input_id(&self) -> Self { let parent_type = match self.circuit_type { ProvingJobCircuitType::RegisterUser => ProvingJobCircuitType::RegisterUserAggregate, @@ -524,6 +566,12 @@ impl QProvingJobDataID { pub fn to_fixed_bytes(&self) -> QProvingJobDataIDSerialized { self.into() } + pub fn with_task_index(&self, task_index: u32) -> Self { + Self { + task_index, + ..*self + } + } pub fn to_hex_string(&self) -> String { hex::encode(&self.to_fixed_bytes()) } diff --git a/city_rollup_common/src/qworker/job_witnesses/inspect.rs b/city_rollup_common/src/qworker/job_witnesses/inspect.rs new file mode 100644 index 00000000..f112d1a8 --- /dev/null +++ b/city_rollup_common/src/qworker/job_witnesses/inspect.rs @@ -0,0 +1,137 @@ +use city_common::data::u8bytes::U8Bytes; +use city_crypto::hash::merkle::treeprover::{AggStateTransitionInput, AggStateTransitionWithEventsInput, DummyAggStateTransition, DummyAggStateTransitionWithEvents}; +use plonky2::{hash::hash_types::RichField, plonk::{config::GenericConfig, proof::ProofWithPublicInputs}}; +use serde::{Deserialize, Serialize}; + +use crate::qworker::job_id::{ProvingJobCircuitType, QProvingJobDataID}; + +use super::{agg::{CRAggAddProcessL1WithdrawalAddL1DepositCircuitInput, CRAggUserRegisterClaimDepositL2TransferCircuitInput, CRBlockStateTransitionCircuitInput}, op::{CRAddL1DepositCircuitInput, CRAddL1WithdrawalCircuitInput, CRClaimL1DepositCircuitInput, CRL2TransferCircuitInput, CRProcessL1WithdrawalCircuitInput, CRUserRegistrationCircuitInput}, sighash::{CRSigHashFinalGLCircuitInput, CRSigHashWrapperCircuitInput}}; + +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub struct QJobProofPublicInputs { + pub job_id: QProvingJobDataID, + pub public_inputs: Vec, +} +impl QJobProofPublicInputs { + pub fn new(job_id: QProvingJobDataID, public_inputs: Vec) -> Self { + Self { job_id, public_inputs } + } +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub struct QJobProofProofOutput, const D: usize> { + pub job_id: QProvingJobDataID, + pub proof: ProofWithPublicInputs, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub struct QJobWitnessWithId { + pub job_id: QProvingJobDataID, + pub witness: QJobWitness, +} +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +#[serde(tag = "q_witness_type")] +pub enum QJobWitness { + RegisterUser(CRUserRegistrationCircuitInput), + RegisterUserAggregate(AggStateTransitionInput), + AddL1Deposit(CRAddL1DepositCircuitInput), + AddL1DepositAggregate(AggStateTransitionWithEventsInput), + ClaimL1Deposit(CRClaimL1DepositCircuitInput), + ClaimL1DepositAggregate(AggStateTransitionInput), + TransferTokensL2(CRL2TransferCircuitInput), + TransferTokensL2Aggregate(AggStateTransitionInput), + AddL1Withdrawal(CRAddL1WithdrawalCircuitInput), + AddL1WithdrawalAggregate(AggStateTransitionInput), + ProcessL1Withdrawal(CRProcessL1WithdrawalCircuitInput), + ProcessL1WithdrawalAggregate(AggStateTransitionWithEventsInput), + GenerateRollupStateTransitionProof(CRBlockStateTransitionCircuitInput), + GenerateSigHashIntrospectionProof(CRSigHashWrapperCircuitInput), + GenerateFinalSigHashProof(CRSigHashFinalGLCircuitInput), + WrapFinalSigHashProofBLS12381(QProvingJobDataID), + AggUserRegisterClaimDepositL2Transfer(CRAggUserRegisterClaimDepositL2TransferCircuitInput), + AggAddProcessL1WithdrawalAddL1Deposit(CRAggAddProcessL1WithdrawalAddL1DepositCircuitInput), + DummyRegisterUserAggregate(DummyAggStateTransition), + DummyAddL1DepositAggregate(DummyAggStateTransitionWithEvents), + DummyClaimL1DepositAggregate(DummyAggStateTransition), + DummyTransferTokensL2Aggregate(DummyAggStateTransition), + DummyAddL1WithdrawalAggregate(DummyAggStateTransition), + DummyProcessL1WithdrawalAggregate(DummyAggStateTransitionWithEvents), + RawBytes(U8Bytes), +} + +impl QJobWitness { + pub fn can_deserialize_witness(job_id: QProvingJobDataID) -> bool { + match job_id.circuit_type { + ProvingJobCircuitType::RegisterUser => true, + ProvingJobCircuitType::RegisterUserAggregate => true, + ProvingJobCircuitType::AddL1Deposit => true, + ProvingJobCircuitType::AddL1DepositAggregate => true, + ProvingJobCircuitType::ClaimL1Deposit => true, + ProvingJobCircuitType::ClaimL1DepositAggregate => true, + ProvingJobCircuitType::TransferTokensL2 => true, + ProvingJobCircuitType::TransferTokensL2Aggregate => true, + ProvingJobCircuitType::AddL1Withdrawal => true, + ProvingJobCircuitType::AddL1WithdrawalAggregate => true, + ProvingJobCircuitType::ProcessL1Withdrawal => true, + ProvingJobCircuitType::ProcessL1WithdrawalAggregate => true, + ProvingJobCircuitType::GenerateRollupStateTransitionProof => true, + ProvingJobCircuitType::GenerateSigHashIntrospectionProof => true, + ProvingJobCircuitType::GenerateFinalSigHashProof => true, + ProvingJobCircuitType::WrapFinalSigHashProofBLS12381 => true, + ProvingJobCircuitType::AggUserRegisterClaimDepositL2Transfer => true, + ProvingJobCircuitType::AggAddProcessL1WithdrawalAddL1Deposit => true, + ProvingJobCircuitType::DummyRegisterUserAggregate => true, + ProvingJobCircuitType::DummyAddL1DepositAggregate => true, + ProvingJobCircuitType::DummyClaimL1DepositAggregate => true, + ProvingJobCircuitType::DummyTransferTokensL2Aggregate => true, + ProvingJobCircuitType::DummyAddL1WithdrawalAggregate => true, + ProvingJobCircuitType::DummyProcessL1WithdrawalAggregate => true, + _ => false, + } + } + pub fn try_deserialize_witness(job_id: QProvingJobDataID, data: &[u8]) -> anyhow::Result { + match job_id.circuit_type { + ProvingJobCircuitType::RegisterUser => Ok(Self::RegisterUser(bincode::deserialize(data)?)), + ProvingJobCircuitType::RegisterUserAggregate => Ok(Self::RegisterUserAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::AddL1Deposit => Ok(Self::AddL1Deposit(bincode::deserialize(data)?)), + ProvingJobCircuitType::AddL1DepositAggregate => Ok(Self::AddL1DepositAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::ClaimL1Deposit => Ok(Self::ClaimL1Deposit(bincode::deserialize(data)?)), + ProvingJobCircuitType::ClaimL1DepositAggregate => Ok(Self::ClaimL1DepositAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::TransferTokensL2 => Ok(Self::TransferTokensL2(bincode::deserialize(data)?)), + ProvingJobCircuitType::TransferTokensL2Aggregate => Ok(Self::TransferTokensL2Aggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::AddL1Withdrawal => Ok(Self::AddL1Withdrawal(bincode::deserialize(data)?)), + ProvingJobCircuitType::AddL1WithdrawalAggregate => Ok(Self::AddL1WithdrawalAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::ProcessL1Withdrawal => Ok(Self::ProcessL1Withdrawal(bincode::deserialize(data)?)), + ProvingJobCircuitType::ProcessL1WithdrawalAggregate => Ok(Self::ProcessL1WithdrawalAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::GenerateRollupStateTransitionProof => Ok(Self::GenerateRollupStateTransitionProof(bincode::deserialize(data)?)), + ProvingJobCircuitType::GenerateSigHashIntrospectionProof => Ok(Self::GenerateSigHashIntrospectionProof(bincode::deserialize(data)?)), + ProvingJobCircuitType::GenerateFinalSigHashProof => Ok(Self::GenerateFinalSigHashProof(bincode::deserialize(data)?)), + ProvingJobCircuitType::WrapFinalSigHashProofBLS12381 => Ok(Self::WrapFinalSigHashProofBLS12381(bincode::deserialize(data)?)), + ProvingJobCircuitType::AggUserRegisterClaimDepositL2Transfer => Ok(Self::AggUserRegisterClaimDepositL2Transfer(bincode::deserialize(data)?)), + ProvingJobCircuitType::AggAddProcessL1WithdrawalAddL1Deposit => Ok(Self::AggAddProcessL1WithdrawalAddL1Deposit(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyRegisterUserAggregate => Ok(Self::DummyRegisterUserAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyAddL1DepositAggregate => Ok(Self::DummyAddL1DepositAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyClaimL1DepositAggregate => Ok(Self::DummyClaimL1DepositAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyTransferTokensL2Aggregate => Ok(Self::DummyTransferTokensL2Aggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyAddL1WithdrawalAggregate => Ok(Self::DummyAddL1WithdrawalAggregate(bincode::deserialize(data)?)), + ProvingJobCircuitType::DummyProcessL1WithdrawalAggregate => Ok(Self::DummyProcessL1WithdrawalAggregate(bincode::deserialize(data)?)), + _ => Ok(Self::RawBytes(U8Bytes::from(data.to_vec()))), + } + } +} + +impl QJobWitnessWithId { + pub fn try_deserialize_witness(job_id: QProvingJobDataID, data: &[u8]) -> anyhow::Result { + Ok(Self { + job_id, + witness: QJobWitness::try_deserialize_witness(job_id, data)?, + }) + } + pub fn can_deserialize_witness(job_id: QProvingJobDataID) -> bool { + QJobWitness::::can_deserialize_witness(job_id) + } +} \ No newline at end of file diff --git a/city_rollup_common/src/qworker/job_witnesses/mod.rs b/city_rollup_common/src/qworker/job_witnesses/mod.rs index 4d275f7a..4e065fb1 100644 --- a/city_rollup_common/src/qworker/job_witnesses/mod.rs +++ b/city_rollup_common/src/qworker/job_witnesses/mod.rs @@ -1,3 +1,4 @@ pub mod agg; pub mod op; pub mod sighash; +pub mod inspect; \ No newline at end of file diff --git a/city_rollup_common/src/qworker/job_witnesses/op.rs b/city_rollup_common/src/qworker/job_witnesses/op.rs index e7934c48..41019035 100644 --- a/city_rollup_common/src/qworker/job_witnesses/op.rs +++ b/city_rollup_common/src/qworker/job_witnesses/op.rs @@ -32,6 +32,18 @@ pub struct CircuitInputWithJobId AggStateTrackableInput for DummyCircuitInputWithJobId { + fn get_state_transition(&self) -> AggStateTransition { + AggStateTransition{ + state_transition_start: QHashOut::ZERO, + state_transition_end: QHashOut::ZERO, + } + } +} #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] #[serde(bound = "")] diff --git a/city_rollup_common/src/qworker/memory_proof_store/mod.rs b/city_rollup_common/src/qworker/memory_proof_store/mod.rs index a924ce46..22f760af 100644 --- a/city_rollup_common/src/qworker/memory_proof_store/mod.rs +++ b/city_rollup_common/src/qworker/memory_proof_store/mod.rs @@ -1,12 +1,13 @@ use std::collections::HashMap; use plonky2::plonk::{config::GenericConfig, proof::ProofWithPublicInputs}; +use serde::{Deserialize, Serialize}; use super::{ job_id::QProvingJobDataID, proof_store::{QProofStoreReaderSync, QProofStoreWriterSync}, }; - +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct SimpleProofStoreMemory { pub proofs: HashMap>, pub counters: HashMap, @@ -18,6 +19,12 @@ impl SimpleProofStoreMemory { counters: HashMap::new(), } } + pub fn to_serialized_bytes(&self) -> anyhow::Result> { + bincode::serialize(self).map_err(|err| anyhow::anyhow!("{}", err)) + } + pub fn from_serialized_bytes(data: &[u8]) -> anyhow::Result { + bincode::deserialize(data).map_err(|err| anyhow::anyhow!("{}", err)) + } } impl QProofStoreReaderSync for SimpleProofStoreMemory { @@ -76,4 +83,20 @@ impl QProofStoreWriterSync for SimpleProofStoreMemory { self.proofs.insert(id, data.to_vec()); Ok(()) } + + fn write_next_jobs( + &mut self, + jobs: &[QProvingJobDataID], + next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + self.write_next_jobs_core(jobs, next_jobs) + } + + fn write_multidimensional_jobs( + &mut self, + jobs_levels: &[Vec], + next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + self.write_multidimensional_jobs_core(jobs_levels, next_jobs) + } } diff --git a/city_rollup_common/src/qworker/mod.rs b/city_rollup_common/src/qworker/mod.rs index 5e7dc7f2..86e875ac 100644 --- a/city_rollup_common/src/qworker/mod.rs +++ b/city_rollup_common/src/qworker/mod.rs @@ -4,3 +4,4 @@ pub mod job_witnesses; pub mod memory_proof_store; pub mod proof_store; pub mod verifier; +pub mod dump; \ No newline at end of file diff --git a/city_rollup_common/src/qworker/proof_store.rs b/city_rollup_common/src/qworker/proof_store.rs index 97f7e7fb..947a7fb7 100644 --- a/city_rollup_common/src/qworker/proof_store.rs +++ b/city_rollup_common/src/qworker/proof_store.rs @@ -3,12 +3,30 @@ use plonky2::plonk::{config::GenericConfig, proof::ProofWithPublicInputs}; use super::job_id::QProvingJobDataID; + + pub trait QProofStoreReaderSync { fn get_proof_by_id, const D: usize>( &self, id: QProvingJobDataID, ) -> anyhow::Result>; fn get_bytes_by_id(&self, id: QProvingJobDataID) -> anyhow::Result>; + fn get_goal_by_job_id(&self, id: QProvingJobDataID) -> anyhow::Result { + let counter_id = id.get_sub_group_counter_id(); + let goal_id = counter_id.get_sub_group_counter_goal_id(); + //tracing::info!("goal_id: {:?}", goal_id); + let goal = self.get_bytes_by_id(goal_id)?; + Ok(u32::from_le_bytes(goal.try_into().unwrap())) + } + fn get_next_jobs_by_job_id( + &self, + id: QProvingJobDataID, + ) -> anyhow::Result> { + let counter_id = id.get_sub_group_counter_id(); + let next_jobs_id = counter_id.get_sub_group_counter_goal_next_jobs_id(); + let next_jobs = self.get_bytes_by_id(next_jobs_id)?; + Ok(bincode::deserialize(&next_jobs)?) + } } pub trait QProofStoreWriterSync { @@ -24,6 +42,11 @@ pub trait QProofStoreWriterSync { &mut self, jobs: &[QProvingJobDataID], next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()>; + fn write_next_jobs_core( + &mut self, + jobs: &[QProvingJobDataID], + next_jobs: &[QProvingJobDataID], ) -> anyhow::Result<()> { let counter_id = jobs[0].get_sub_group_counter_id(); let goal_id = counter_id.get_sub_group_counter_goal_id(); @@ -33,10 +56,16 @@ pub trait QProofStoreWriterSync { self.set_bytes_by_id(next_jobs_id, &bincode::serialize(next_jobs)?)?; Ok(()) } + fn write_multidimensional_jobs( &mut self, jobs_levels: &[Vec], next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()>; + fn write_multidimensional_jobs_core( + &mut self, + jobs_levels: &[Vec], + next_jobs: &[QProvingJobDataID], ) -> anyhow::Result<()> { let job_levels_count = jobs_levels.len(); for i in 0..job_levels_count { @@ -59,22 +88,7 @@ pub trait QProofStoreWriterSync { } pub trait QProofStore: QProofStoreReaderSync + QProofStoreWriterSync { - fn get_goal_by_job_id(&self, id: QProvingJobDataID) -> anyhow::Result { - let counter_id = id.get_sub_group_counter_id(); - let goal_id = counter_id.get_sub_group_counter_goal_id(); - //tracing::info!("goal_id: {:?}", goal_id); - let goal = self.get_bytes_by_id(goal_id)?; - Ok(u32::from_le_bytes(goal.try_into().unwrap())) - } - fn get_next_jobs_by_job_id( - &self, - id: QProvingJobDataID, - ) -> anyhow::Result> { - let counter_id = id.get_sub_group_counter_id(); - let next_jobs_id = counter_id.get_sub_group_counter_goal_next_jobs_id(); - let next_jobs = self.get_bytes_by_id(next_jobs_id)?; - Ok(bincode::deserialize(&next_jobs)?) - } + } impl QProofStore for T {} @@ -99,3 +113,59 @@ pub trait QProofStoreWriterAsync { id: QProvingJobDataID, ) -> anyhow::Result; } + + +#[derive(Clone, Copy, Debug)] +pub struct QDummyProofStore {} + +impl QDummyProofStore { + pub fn new() -> Self { + Self {} + } +} + +impl QProofStoreReaderSync for QDummyProofStore { + fn get_proof_by_id, const D: usize>( + &self, + _id: QProvingJobDataID, + ) -> anyhow::Result> { + anyhow::bail!("Not implemented") + } + + fn get_bytes_by_id(&self, _id: QProvingJobDataID) -> anyhow::Result> { + Ok(vec![]) + } +} +impl QProofStoreWriterSync for QDummyProofStore { + fn set_proof_by_id, const D: usize>( + &mut self, + _id: QProvingJobDataID, + _proof: &ProofWithPublicInputs, + ) -> anyhow::Result<()> { + anyhow::bail!("Not implemented") + } + + fn set_bytes_by_id(&mut self, _id: QProvingJobDataID, _data: &[u8]) -> anyhow::Result<()> { + anyhow::bail!("Not implemented") + } + + fn inc_counter_by_id(&mut self, _id: QProvingJobDataID) -> anyhow::Result { + anyhow::bail!("Not implemented") + } + + fn write_next_jobs( + &mut self, + _jobs: &[QProvingJobDataID], + _next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + anyhow::bail!("Not implemented") + } + + fn write_multidimensional_jobs( + &mut self, + _jobs_levels: &[Vec], + _next_jobs: &[QProvingJobDataID], + ) -> anyhow::Result<()> { + anyhow::bail!("Not implemented") + } +} \ No newline at end of file diff --git a/city_rollup_core_api/Cargo.toml b/city_rollup_core_api/Cargo.toml index a95ecbbd..dc41743c 100644 --- a/city_rollup_core_api/Cargo.toml +++ b/city_rollup_core_api/Cargo.toml @@ -8,10 +8,11 @@ jsonrpsee = { workspace = true } tokio = { workspace = true } anyhow = { workspace = true } redb = { workspace = true } -kvq_store_rocksdb = { path = "../kvq_store_rocksdb" } +kvq_store_redb = { path = "../kvq_store_redb" } city_store = { path = "../city_store" } city_common = { path = "../city_common" } city_rollup_common = { path = "../city_rollup_common" } +city_macros = { path = "../city_macros" } city_crypto = { path = "../city_crypto" } hex = { workspace = true } futures = { workspace = true } diff --git a/city_rollup_core_api/src/lib.rs b/city_rollup_core_api/src/lib.rs index 858bbf7f..fa519334 100644 --- a/city_rollup_core_api/src/lib.rs +++ b/city_rollup_core_api/src/lib.rs @@ -1,16 +1,25 @@ +use std::sync::Arc; +use city_common::data::kv::SimpleKVPair; +use city_common::data::u8bytes::U8Bytes; use city_crypto::hash::base_types::hash160::Hash160; use city_crypto::hash::base_types::hash256::Hash256; +use city_macros::define_table; use city_rollup_common::api::data::store::{ - CityL1Deposit, CityL1Withdrawal, CityL2BlockState, CityUserState, + CityL1DepositJSON, CityL1Withdrawal, CityL2BlockState, CityUserState, }; +use city_rollup_common::qworker::job_id::{QProvingJobDataID, QProvingJobDataIDSerializedWrapped}; +use city_rollup_common::qworker::proof_store::QProofStoreReaderSync; use city_store::config::{CityHash, CityMerkleProof}; use city_store::store::city::base::CityStore; use jsonrpsee::core::async_trait; use jsonrpsee::proc_macros::rpc; use jsonrpsee::server::Server; use jsonrpsee::types::{ErrorCode, ErrorObject, ErrorObjectOwned}; -use kvq_store_rocksdb::KVQRocksDBStore; +use kvq_store_redb::KVQReDBStore; +use redb::{Database, ReadOnlyTable, TableDefinition}; + +define_table! { KV, &[u8], &[u8] } #[rpc(server, client, namespace = "cr")] pub trait Rpc { @@ -60,26 +69,26 @@ pub trait Rpc { &self, checkpoint_id: u64, deposit_id: u64, - ) -> Result; + ) -> Result; #[method(name = "getDepositsById")] async fn get_deposits_by_id( &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> Result, ErrorObjectOwned>; + ) -> Result, ErrorObjectOwned>; #[method(name = "getDepositByTxid")] async fn get_deposit_by_txid( &self, transaction_id: Hash256, - ) -> Result; + ) -> Result; #[method(name = "getDepositsByTxid")] async fn get_deposits_by_txid( &self, transaction_ids: Vec, - ) -> Result, ErrorObjectOwned>; + ) -> Result, ErrorObjectOwned>; #[method(name = "getDepositHash")] async fn get_deposit_hash( @@ -155,17 +164,43 @@ pub trait Rpc { checkpoint_id: u64, withdrawal_id: u64, ) -> Result; + + #[method(name = "getProofStoreValue")] + async fn get_proof_store_value( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> Result; + + #[method(name = "getProofStoreValues")] + async fn get_proof_store_values( + &self, + keys: Vec, + ) -> Result>, ErrorObjectOwned>; } #[derive(Clone)] -pub struct RpcServerImpl { - db: KVQRocksDBStore, +pub struct RpcServerImpl { + db: Arc, + proof_store: PS, +} + +impl RpcServerImpl { + pub fn query_store( + &self, + f: impl FnOnce(KVQReDBStore>) -> anyhow::Result, + ) -> anyhow::Result { + let rxn = self.db.begin_read()?; + let table = rxn.open_table(KV)?; + + f(KVQReDBStore::new(table)) + } } #[async_trait] -impl RpcServer for RpcServerImpl { +impl RpcServer for RpcServerImpl { async fn get_user_tree_root(&self, checkpoint_id: u64) -> Result { - Ok(CityStore::get_user_tree_root(&self.db, checkpoint_id) + Ok(self + .query_store(|store| Ok(CityStore::get_user_tree_root(&store, checkpoint_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } @@ -174,7 +209,8 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, user_id: u64, ) -> Result { - Ok(CityStore::get_user_by_id(&self.db, checkpoint_id, user_id) + Ok(self + .query_store(|store| Ok(CityStore::get_user_by_id(&store, checkpoint_id, user_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } @@ -182,7 +218,8 @@ impl RpcServer for RpcServerImpl { &self, public_key: CityHash, ) -> Result, ErrorObjectOwned> { - Ok(CityStore::get_user_ids_for_public_key(&self.db, public_key) + Ok(self + .query_store(|store| Ok(CityStore::get_user_ids_for_public_key(&store, public_key)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } @@ -191,10 +228,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, user_id: u64, ) -> Result { - Ok( - CityStore::get_user_merkle_proof_by_id(&self.db, checkpoint_id, user_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_user_merkle_proof_by_id( + &store, + checkpoint_id, + user_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_user_tree_leaf( @@ -202,10 +244,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, leaf_id: u64, ) -> Result { - Ok( - CityStore::get_user_tree_leaf(&self.db, checkpoint_id, leaf_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_user_tree_leaf( + &store, + checkpoint_id, + leaf_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_user_tree_leaf_merkle_proof( @@ -213,17 +260,23 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, leaf_id: u64, ) -> Result { - Ok( - CityStore::get_user_tree_leaf_merkle_proof(&self.db, checkpoint_id, leaf_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_user_tree_leaf_merkle_proof( + &store, + checkpoint_id, + leaf_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_deposit_tree_root( &self, checkpoint_id: u64, ) -> Result { - Ok(CityStore::get_deposit_tree_root(&self.db, checkpoint_id) + Ok(self + .query_store(|store| Ok(CityStore::get_deposit_tree_root(&store, checkpoint_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } @@ -231,38 +284,71 @@ impl RpcServer for RpcServerImpl { &self, checkpoint_id: u64, deposit_id: u64, - ) -> Result { - Ok( - CityStore::get_deposit_by_id(&self.db, checkpoint_id, deposit_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + ) -> Result { + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposit_by_id( + &store, + checkpoint_id, + deposit_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))? + .to_json_variant()) } async fn get_deposits_by_id( &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> Result, ErrorObjectOwned> { - Ok( - CityStore::get_deposits_by_id(&self.db, checkpoint_id, &deposit_ids) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + ) -> Result, ErrorObjectOwned> { + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposits_by_id( + &store, + checkpoint_id, + &deposit_ids, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))? + .into_iter() + .map(|x| x.to_json_variant()) + .collect::>()) } async fn get_deposit_by_txid( &self, transaction_id: Hash256, - ) -> Result { - Ok(CityStore::get_deposit_by_txid(&self.db, transaction_id.reversed()) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) + ) -> Result { + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposit_by_txid( + &store, + transaction_id.reversed(), + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))? + .to_json_variant()) } async fn get_deposits_by_txid( &self, transaction_ids: Vec, - ) -> Result, ErrorObjectOwned> { - Ok(CityStore::get_deposits_by_txid(&self.db, &transaction_ids.into_iter().map(|x|x.reversed()).collect::>()) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) + ) -> Result, ErrorObjectOwned> { + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposits_by_txid( + &store, + &transaction_ids + .into_iter() + .map(|x| x.reversed()) + .collect::>(), + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))? + .into_iter() + .map(|x| x.to_json_variant()) + .collect::>()) } async fn get_deposit_hash( @@ -270,10 +356,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, deposit_id: u64, ) -> Result { - Ok( - CityStore::get_deposit_hash(&self.db, checkpoint_id, deposit_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposit_hash( + &store, + checkpoint_id, + deposit_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_deposit_leaf_merkle_proof( @@ -281,33 +372,42 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, deposit_id: u64, ) -> Result { - Ok( - CityStore::get_deposit_leaf_merkle_proof(&self.db, checkpoint_id, deposit_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_deposit_leaf_merkle_proof( + &store, + checkpoint_id, + deposit_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_block_state( &self, checkpoint_id: u64, ) -> Result { - Ok(CityStore::get_block_state(&self.db, checkpoint_id) + Ok(self + .query_store(|store| Ok(CityStore::get_block_state(&store, checkpoint_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_latest_block_state(&self) -> Result { - Ok(CityStore::get_latest_block_state(&self.db) + Ok(self + .query_store(|store| Ok(CityStore::get_latest_block_state(&store)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_city_root(&self, checkpoint_id: u64) -> Result { - Ok(CityStore::get_city_root(&self.db, checkpoint_id) + Ok(self + .query_store(|store| Ok(CityStore::get_city_root(&store, checkpoint_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_city_block_script(&self, checkpoint_id: u64) -> Result { Ok(hex::encode( - &CityStore::get_city_block_script(&self.db, checkpoint_id) + &self + .query_store(|store| Ok(CityStore::get_city_block_script(&store, checkpoint_id)?)) .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, )) } @@ -316,30 +416,37 @@ impl RpcServer for RpcServerImpl { &self, checkpoint_id: u64, ) -> Result { - Ok( - CityStore::get_city_block_deposit_address(&self.db, checkpoint_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_city_block_deposit_address( + &store, + checkpoint_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_city_block_deposit_address_string( &self, checkpoint_id: u64, ) -> Result { - Ok( - CityStore::get_city_block_deposit_address_string(&self.db, checkpoint_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_city_block_deposit_address_string( + &store, + checkpoint_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_withdrawal_tree_root( &self, checkpoint_id: u64, ) -> Result { - Ok( - CityStore::get_withdrawal_tree_root(&self.db, checkpoint_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| Ok(CityStore::get_withdrawal_tree_root(&store, checkpoint_id)?)) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_withdrawal_by_id( @@ -347,10 +454,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, withdrawal_id: u64, ) -> Result { - Ok( - CityStore::get_withdrawal_by_id(&self.db, checkpoint_id, withdrawal_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_withdrawal_by_id( + &store, + checkpoint_id, + withdrawal_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_withdrawals_by_id( @@ -358,10 +470,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, withdrawal_ids: Vec, ) -> Result, ErrorObjectOwned> { - Ok( - CityStore::get_withdrawals_by_id(&self.db, checkpoint_id, &withdrawal_ids) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_withdrawals_by_id( + &store, + checkpoint_id, + &withdrawal_ids, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_withdrawal_hash( @@ -369,10 +486,15 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, withdrawal_id: u64, ) -> Result { - Ok( - CityStore::get_withdrawal_hash(&self.db, checkpoint_id, withdrawal_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_withdrawal_hash( + &store, + checkpoint_id, + withdrawal_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) } async fn get_withdrawal_leaf_merkle_proof( @@ -380,17 +502,59 @@ impl RpcServer for RpcServerImpl { checkpoint_id: u64, withdrawal_id: u64, ) -> Result { - Ok( - CityStore::get_withdrawal_leaf_merkle_proof(&self.db, checkpoint_id, withdrawal_id) - .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?, - ) + Ok(self + .query_store(|store| { + Ok(CityStore::get_withdrawal_leaf_merkle_proof( + &store, + checkpoint_id, + withdrawal_id, + )?) + }) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?) + } + + async fn get_proof_store_value( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> Result { + let result = self + .proof_store + .get_bytes_by_id( + QProvingJobDataID::try_from(key.0) + .map_err(|_| ErrorObject::from(ErrorCode::InvalidParams))?, + ) + .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?; + + Ok(U8Bytes(result)) + } + async fn get_proof_store_values( + &self, + keys: Vec, + ) -> Result>, ErrorObjectOwned> + { + let id_keys = keys + .iter() + .map(|x| { + QProvingJobDataID::try_from(x.0) + .map_err(|_| ErrorObject::from(ErrorCode::InvalidParams)) + }) + .collect::, ErrorObjectOwned>>()?; + + id_keys.iter().map(|key|{ + let result = self.proof_store.get_bytes_by_id(*key).map_err(|_| ErrorObject::from(ErrorCode::InternalError))?; + Ok(SimpleKVPair{key: QProvingJobDataIDSerializedWrapped(key.to_fixed_bytes()), value: U8Bytes(result)}) + }).collect::>, ErrorObjectOwned>>() } } -pub async fn run_server(server_addr: String, db: KVQRocksDBStore) -> anyhow::Result<()> { +pub async fn run_server( + server_addr: String, + db: Arc, + proof_store: PS, +) -> anyhow::Result<()> { let server = Server::builder().build(server_addr).await?; - let rpc_server_impl = RpcServerImpl { db }; + let rpc_server_impl = RpcServerImpl { db, proof_store }; let handle = server.start(rpc_server_impl.into_rpc()); tokio::spawn(handle.stopped()); Ok(futures::future::pending::<()>().await) diff --git a/city_rollup_core_node/Cargo.toml b/city_rollup_core_node/Cargo.toml index 4b61deb2..8347a8ca 100644 --- a/city_rollup_core_node/Cargo.toml +++ b/city_rollup_core_node/Cargo.toml @@ -35,6 +35,9 @@ hyper-util = { workspace = true } jsonrpsee = { workspace = true } tracing = { workspace = true } +tower = { workspace = true } +tower-http = { workspace = true } + [dev-dependencies] criterion = "0.5.1" rand_chacha = "0.3.1" diff --git a/city_rollup_core_node/src/handler.rs b/city_rollup_core_node/src/handler.rs index 305a2349..a3822e8d 100644 --- a/city_rollup_core_node/src/handler.rs +++ b/city_rollup_core_node/src/handler.rs @@ -75,6 +75,7 @@ impl CityRollupRPCServerHandler { match (req.method(), req.uri().path()) { (&Method::GET, "/editor") => Ok(editor()), (&Method::POST, "/") => self.rpc(req).await, + (&Method::OPTIONS, "/") => self.preflight(req).await, _ => Ok(not_found()), } } @@ -136,7 +137,16 @@ impl CityRollupRPCServerHandler { .header(header::CONTENT_TYPE, "application/json") .body(full(serde_json::to_vec(&response)?))?) } - + pub async fn preflight(&self, req: Request) -> anyhow::Result> { + let _whole_body = req.collect().await?; + let response = Response::builder() + .status(StatusCode::OK) + .header("Access-Control-Allow-Origin", "*") + .header("Access-Control-Allow-Headers", "*") + .header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + .body(BoxBody::default())?; + Ok(response) + } fn register_user(&mut self, req: CityRegisterUserRPCRequest) -> Result<(), anyhow::Error> { self.notify_rpc_register_user(&req)?; Ok(()) diff --git a/city_rollup_core_orchestrator/Cargo.toml b/city_rollup_core_orchestrator/Cargo.toml index 849042bb..f5f0cbac 100644 --- a/city_rollup_core_orchestrator/Cargo.toml +++ b/city_rollup_core_orchestrator/Cargo.toml @@ -31,7 +31,8 @@ redis = { workspace = true } city_macros = { path = "../city_macros" } city_redis_store = { path = "../city_redis_store" } city_rollup_circuit = { path = "../city_rollup_circuit" } -kvq_store_rocksdb = { path = "../kvq_store_rocksdb" } +kvq_store_redb = { path = "../kvq_store_redb" } +redb = { workspace = true } hex-literal = { workspace = true } tracing = { workspace = true } diff --git a/city_rollup_core_orchestrator/src/debug/scenario/actors/job_planner.rs b/city_rollup_core_orchestrator/src/debug/scenario/actors/job_planner.rs new file mode 100644 index 00000000..65d500df --- /dev/null +++ b/city_rollup_core_orchestrator/src/debug/scenario/actors/job_planner.rs @@ -0,0 +1,154 @@ +use city_rollup_common::qworker::{job_id::QProvingJobDataID, proof_store::QProofStore}; + +use crate::debug::scenario::block_planner::transition::CityOpJobIds; + +pub fn plan_jobs( + proof_store: &mut PS, + block_op_job_ids: &CityOpJobIds, + num_input_witnesses: usize, + checkpoint_id: u64, +) -> anyhow::Result> { + let root_state_transition = + QProvingJobDataID::block_state_transition_input_witness(checkpoint_id); + + let agg_jobs_for_inputs: Vec = (0..num_input_witnesses) + .map(|i| QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, 1, i as u32)) + .collect::>(); + + proof_store.write_next_jobs( + &agg_jobs_for_inputs, + &[QProvingJobDataID::notify_block_complete(checkpoint_id)], + )?; + + let per_input_jobs = (0..num_input_witnesses) + .map(|i| { + ( + QProvingJobDataID::wrap_sighash_final_bls3812_input_witness(checkpoint_id, i), + QProvingJobDataID::sighash_final_input_witness(checkpoint_id, i), + QProvingJobDataID::sighash_introspection_input_witness(checkpoint_id, i), + ) + }) + .collect::>(); + + for (i, pij) in per_input_jobs.iter().enumerate() { + proof_store.write_next_jobs(&[pij.0], &[agg_jobs_for_inputs[i]])?; + proof_store.write_next_jobs(&[pij.1], &[pij.0])?; + } + let agg_state_and_introspections_group_id = 5; + let agg_state_root_id = QProvingJobDataID::get_block_aggregate_jobs_group( + checkpoint_id, + agg_state_and_introspections_group_id, + 0, + ); + let agg_all_introspections_ids = QProvingJobDataID::get_block_aggregate_jobs_group( + checkpoint_id, + agg_state_and_introspections_group_id, + 1, + ); + let introspection_jobs = per_input_jobs.iter().map(|x| x.2).collect::>(); + proof_store.write_next_jobs(&introspection_jobs, &[agg_all_introspections_ids])?; + let final_input_witness_jobs = per_input_jobs.iter().map(|x| x.1).collect::>(); + proof_store.write_next_jobs( + &[agg_state_root_id, agg_all_introspections_ids], + &final_input_witness_jobs, + )?; + + proof_store.write_next_jobs(&[root_state_transition], &[agg_state_root_id])?; + + let op_agg_group_parts_common_id = 6; + + let state_part_1_common_id = QProvingJobDataID::get_block_aggregate_jobs_group( + checkpoint_id, + op_agg_group_parts_common_id, + 0, + ); + let state_part_2_common_id = QProvingJobDataID::get_block_aggregate_jobs_group( + checkpoint_id, + op_agg_group_parts_common_id, + 1, + ); + + let state_part_1_id = QProvingJobDataID::block_agg_state_part_1_input_witness(checkpoint_id); + let state_part_2_id = QProvingJobDataID::block_agg_state_part_2_input_witness(checkpoint_id); + + proof_store.write_next_jobs( + &[state_part_1_common_id, state_part_2_common_id], + &[root_state_transition], + )?; + + proof_store.write_next_jobs(&[state_part_1_id], &[state_part_1_common_id])?; + proof_store.write_next_jobs(&[state_part_2_id], &[state_part_2_common_id])?; + + let op_agg_group_part_1_id = 11; + let register_users_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_1_id, 0); + let claim_deposits_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_1_id, 1); + let transfer_tokens_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_1_id, 2); + + proof_store.write_next_jobs( + &[ + register_users_agg_job_id, + claim_deposits_agg_job_id, + transfer_tokens_agg_job_id, + ], + &[state_part_1_id], + )?; + + let op_agg_group_part_2_id = 12; + let add_withdrawals_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_2_id, 0); + let process_withdrawals_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_2_id, 1); + let add_deposits_agg_job_id = + QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, op_agg_group_part_2_id, 2); + + proof_store.write_next_jobs( + &[ + add_withdrawals_agg_job_id, + process_withdrawals_agg_job_id, + add_deposits_agg_job_id, + ], + &[state_part_2_id], + )?; + + proof_store.write_multidimensional_jobs( + &block_op_job_ids.register_user_job_ids, + &[register_users_agg_job_id], + )?; + proof_store.write_multidimensional_jobs( + &block_op_job_ids.claim_deposit_job_ids, + &[claim_deposits_agg_job_id], + )?; + proof_store.write_multidimensional_jobs( + &block_op_job_ids.token_transfer_job_ids, + &[transfer_tokens_agg_job_id], + )?; + + proof_store.write_multidimensional_jobs( + &block_op_job_ids.add_withdrawal_job_ids, + &[add_withdrawals_agg_job_id], + )?; + proof_store.write_multidimensional_jobs( + &block_op_job_ids.process_withdrawal_job_ids, + &[process_withdrawals_agg_job_id], + )?; + proof_store.write_multidimensional_jobs( + &block_op_job_ids.add_deposit_job_ids, + &[add_deposits_agg_job_id], + )?; + + let leaf_jobs = [ + introspection_jobs, + block_op_job_ids.register_user_job_ids[0].to_vec(), + block_op_job_ids.claim_deposit_job_ids[0].to_vec(), + block_op_job_ids.token_transfer_job_ids[0].to_vec(), + block_op_job_ids.add_withdrawal_job_ids[0].to_vec(), + block_op_job_ids.process_withdrawal_job_ids[0].to_vec(), + block_op_job_ids.add_deposit_job_ids[0].to_vec(), + ] + .concat(); + + Ok(leaf_jobs) +} diff --git a/city_rollup_core_orchestrator/src/debug/scenario/actors/mod.rs b/city_rollup_core_orchestrator/src/debug/scenario/actors/mod.rs index b252f36b..8d4ecc38 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/actors/mod.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/actors/mod.rs @@ -1 +1,2 @@ pub mod simple; +pub mod job_planner; \ No newline at end of file diff --git a/city_rollup_core_orchestrator/src/debug/scenario/actors/simple.rs b/city_rollup_core_orchestrator/src/debug/scenario/actors/simple.rs index db13388c..7c6ce5a4 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/actors/simple.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/actors/simple.rs @@ -2,7 +2,7 @@ use city_common::{ config::rollup_constants::{BLOCK_SCRIPT_SPEND_BASE_FEE_AMOUNT, WITHDRAWAL_FEE_AMOUNT}, logging::debug_timer::DebugTimer, }; -use city_crypto::hash::base_types::{hash160::Hash160, hash256::Hash256}; +use city_crypto::hash::base_types::{felt252::felt252_hashout_to_hash256_le, hash160::Hash160, hash256::Hash256}; use city_rollup_common::{ actors::{ requested_actions::CityScenarioRequestedActions, @@ -19,8 +19,7 @@ use city_rollup_common::{ }, link::{data::BTCAddress160, traits::QBitcoinAPISync}, qworker::{ - fingerprints::CRWorkerToolboxCoreCircuitFingerprints, job_id::QProvingJobDataID, - proof_store::QProofStore, + fingerprints::CRWorkerToolboxCoreCircuitFingerprints, job_id::{QProvingJobDataID, QProvingJobDataIDSerializedWrapped}, proof_store::QProofStore }, }; use city_store::{ @@ -31,7 +30,7 @@ use kvq::traits::KVQBinaryStore; use serde::{Deserialize, Serialize}; use crate::debug::scenario::{ - block_planner::planner::CityOrchestratorBlockPlanner, sighash::finalizer::SigHashFinalizer, + actors::job_planner::plan_jobs, block_planner::planner::CityOrchestratorBlockPlanner, sighash::finalizer::SigHashFinalizer }; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct SimpleActorOrchestratorProduceBlockStep1Result { @@ -113,6 +112,12 @@ pub fn create_hints_for_block( funding_transactions: all_inputs.to_vec(), next_block_redeem_script: next_script.to_vec(), }; + + //tracing::info!("hint[{}]: {}",i, serde_json::to_string(&hint).unwrap()); + tracing::info!("sighash[{}]: {}", i, hint.sighash_preimage.get_hash().to_hex_string()); + //tracing::info!("sighash_252[{}]: {:?}", hint.sighash_preimage.get_hash_felt252::()); + tracing::info!("sighash_252_hex[{}]: {}", i, felt252_hashout_to_hash256_le(hint.sighash_preimage.get_hash_felt252::()).to_hex_string()); + spend_hints.push(hint); } @@ -313,132 +318,9 @@ impl SimpleActorOrchestrator { &all_inputs, &withdrawals, )?; - let tpl_transaction = hints[0].sighash_preimage.transaction.clone(); - let num_input_witnesses = all_inputs.len(); // 1 dummy, but also need the last block - let agg_jobs_for_inputs: Vec = (0..num_input_witnesses) - .map(|i| QProvingJobDataID::get_block_aggregate_jobs_group(checkpoint_id, 1, i as u32)) - .collect::>(); - - proof_store.write_next_jobs( - &agg_jobs_for_inputs, - &[QProvingJobDataID::notify_block_complete(checkpoint_id)], - )?; - - let per_input_jobs = (0..num_input_witnesses) - .map(|i| { - ( - QProvingJobDataID::wrap_sighash_final_bls3812_input_witness(checkpoint_id, i), - QProvingJobDataID::sighash_final_input_witness(checkpoint_id, i), - QProvingJobDataID::sighash_introspection_input_witness(checkpoint_id, i), - ) - }) - .collect::>(); - - for (i, pij) in per_input_jobs.iter().enumerate() { - proof_store.write_next_jobs(&[pij.0], &[agg_jobs_for_inputs[i]])?; - proof_store.write_next_jobs(&[pij.1], &[pij.0])?; - } - let agg_state_and_introspections_group_id = 5; - let agg_state_root_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - agg_state_and_introspections_group_id, - 0, - ); - let agg_all_introspections_ids = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - agg_state_and_introspections_group_id, - 1, - ); - let introspection_jobs = per_input_jobs.iter().map(|x| x.2).collect::>(); - proof_store.write_next_jobs(&introspection_jobs, &[agg_all_introspections_ids])?; - let final_input_witness_jobs = per_input_jobs.iter().map(|x| x.1).collect::>(); - proof_store.write_next_jobs( - &[agg_state_root_id, agg_all_introspections_ids], - &final_input_witness_jobs, - )?; let root_state_transition = QProvingJobDataID::block_state_transition_input_witness(checkpoint_id); - proof_store.write_next_jobs(&[root_state_transition], &[agg_state_root_id])?; - - let op_agg_group_parts_common_id = 6; - - let state_part_1_common_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_parts_common_id, - 0, - ); - let state_part_2_common_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_parts_common_id, - 1, - ); - - let state_part_1_id = - QProvingJobDataID::block_agg_state_part_1_input_witness(checkpoint_id); - let state_part_2_id = - QProvingJobDataID::block_agg_state_part_2_input_witness(checkpoint_id); - - proof_store.write_next_jobs( - &[state_part_1_common_id, state_part_2_common_id], - &[root_state_transition], - )?; - - proof_store.write_next_jobs(&[state_part_1_id], &[state_part_1_common_id])?; - proof_store.write_next_jobs(&[state_part_2_id], &[state_part_2_common_id])?; - - let op_agg_group_part_1_id = 11; - let register_users_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_1_id, - 0, - ); - let claim_deposits_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_1_id, - 1, - ); - let transfer_tokens_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_1_id, - 2, - ); - - proof_store.write_next_jobs( - &[ - register_users_agg_job_id, - claim_deposits_agg_job_id, - transfer_tokens_agg_job_id, - ], - &[state_part_1_id], - )?; - - let op_agg_group_part_2_id = 12; - let add_withdrawals_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_2_id, - 0, - ); - let process_withdrawals_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_2_id, - 1, - ); - let add_deposits_agg_job_id = QProvingJobDataID::get_block_aggregate_jobs_group( - checkpoint_id, - op_agg_group_part_2_id, - 2, - ); - - proof_store.write_next_jobs( - &[ - add_withdrawals_agg_job_id, - process_withdrawals_agg_job_id, - add_deposits_agg_job_id, - ], - &[state_part_2_id], - )?; - let _ = SigHashFinalizer::finalize_sighashes::( proof_store, sighash_whitelist_tree, @@ -446,43 +328,15 @@ impl SimpleActorOrchestrator { root_state_transition, &hints, )?; + let tpl_transaction = hints[0].sighash_preimage.transaction.clone(); + let num_input_witnesses = all_inputs.len(); // 1 dummy, but also need the last block + let leaf_jobs = plan_jobs(proof_store, &block_op_job_ids, num_input_witnesses, checkpoint_id)?; - proof_store.write_multidimensional_jobs( - &block_op_job_ids.register_user_job_ids, - &[register_users_agg_job_id], - )?; - proof_store.write_multidimensional_jobs( - &block_op_job_ids.claim_deposit_job_ids, - &[claim_deposits_agg_job_id], - )?; - proof_store.write_multidimensional_jobs( - &block_op_job_ids.token_transfer_job_ids, - &[transfer_tokens_agg_job_id], - )?; + let leaf_jobs_debug_serialized = leaf_jobs.iter().map(|x|QProvingJobDataIDSerializedWrapped(x.to_fixed_bytes())).collect::>(); + println!("leaf_jobs: {}", serde_json::to_string(&leaf_jobs_debug_serialized)?); + - proof_store.write_multidimensional_jobs( - &block_op_job_ids.add_withdrawal_job_ids, - &[add_withdrawals_agg_job_id], - )?; - proof_store.write_multidimensional_jobs( - &block_op_job_ids.process_withdrawal_job_ids, - &[process_withdrawals_agg_job_id], - )?; - proof_store.write_multidimensional_jobs( - &block_op_job_ids.add_deposit_job_ids, - &[add_deposits_agg_job_id], - )?; - let leaf_jobs = [ - introspection_jobs, - block_op_job_ids.register_user_job_ids[0].to_vec(), - block_op_job_ids.claim_deposit_job_ids[0].to_vec(), - block_op_job_ids.token_transfer_job_ids[0].to_vec(), - block_op_job_ids.add_withdrawal_job_ids[0].to_vec(), - block_op_job_ids.process_withdrawal_job_ids[0].to_vec(), - block_op_job_ids.add_deposit_job_ids[0].to_vec(), - ] - .concat(); Ok(( leaf_jobs, diff --git a/city_rollup_core_orchestrator/src/debug/scenario/block_planner/transition.rs b/city_rollup_core_orchestrator/src/debug/scenario/block_planner/transition.rs index 52992f57..7ca64b48 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/block_planner/transition.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/block_planner/transition.rs @@ -5,7 +5,7 @@ use city_crypto::hash::{ qhashout::QHashOut, }; use city_rollup_common::qworker::{ - job_id::QProvingJobDataID, + job_id::{ProvingJobCircuitType, QProvingJobDataID}, job_witnesses::agg::{ CRAggAddProcessL1WithdrawalAddL1DepositCircuitInput, CRAggUserRegisterClaimDepositL2TransferCircuitInput, @@ -14,6 +14,8 @@ use city_rollup_common::qworker::{ use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; +use super::tree_helper::get_dummy_tree_prover_ids_op_circuit; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(bound = "")] pub struct CityRootStateTransitions { @@ -82,7 +84,15 @@ pub struct CityOpRootJobIds { pub process_withdrawal_job_root_id: QProvingJobDataID, pub add_deposit_job_root_id: QProvingJobDataID, } - +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct CityOpJobConfig { + pub register_user_count: usize, + pub claim_deposit_count: usize, + pub token_transfer_count: usize, + pub add_withdrawal_count: usize, + pub process_withdrawal_count: usize, + pub add_deposit_count: usize, +} #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(bound = "")] pub struct CityOpJobIds { @@ -98,6 +108,53 @@ fn vec_2d_size(arr: &[Vec]) -> usize { arr.iter().map(|x| x.len()).sum() } impl CityOpJobIds { + pub fn dummy_from_config(checkpoint_id: u64, config: &CityOpJobConfig) -> Self { + let register_user_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::RegisterUser, + ProvingJobCircuitType::DummyRegisterUserAggregate, + checkpoint_id, + config.register_user_count, + ); + let claim_deposit_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::ClaimL1Deposit, + ProvingJobCircuitType::DummyClaimL1DepositAggregate, + checkpoint_id, + config.claim_deposit_count, + ); + let token_transfer_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::TransferTokensL2, + ProvingJobCircuitType::DummyTransferTokensL2Aggregate, + checkpoint_id, + config.token_transfer_count, + ); + let add_withdrawal_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::AddL1Withdrawal, + ProvingJobCircuitType::DummyAddL1WithdrawalAggregate, + checkpoint_id, + config.add_withdrawal_count, + ); + let process_withdrawal_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::ProcessL1Withdrawal, + ProvingJobCircuitType::DummyProcessL1WithdrawalAggregate, + checkpoint_id, + config.process_withdrawal_count, + ); + let add_deposit_job_ids = get_dummy_tree_prover_ids_op_circuit( + ProvingJobCircuitType::AddL1Deposit, + ProvingJobCircuitType::DummyAddL1DepositAggregate, + checkpoint_id, + config.add_deposit_count, + ); + Self { + register_user_job_ids, + claim_deposit_job_ids, + token_transfer_job_ids, + add_withdrawal_job_ids, + process_withdrawal_job_ids, + add_deposit_job_ids, + + } + } pub fn get_total_job_ids(&self) -> usize { vec_2d_size(&self.register_user_job_ids) + vec_2d_size(&self.claim_deposit_job_ids) diff --git a/city_rollup_core_orchestrator/src/debug/scenario/block_planner/tree_helper.rs b/city_rollup_core_orchestrator/src/debug/scenario/block_planner/tree_helper.rs index 613a48b8..863eb9cf 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/block_planner/tree_helper.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/block_planner/tree_helper.rs @@ -1,5 +1,6 @@ use std::fmt::Debug; +use city_common::tree_planner::BinaryTreePlanner; use city_crypto::hash::{ merkle::treeprover::{ generate_tree_inputs_with_position, AggStateTrackableInput, @@ -10,13 +11,61 @@ use city_crypto::hash::{ qhashout::QHashOut, }; use city_rollup_common::qworker::{ - job_id::QProvingJobDataID, + job_id::{ProvingJobCircuitType, QProvingJobDataID}, job_witnesses::op::{CircuitInputWithDependencies, CircuitInputWithJobId}, proof_store::QProofStore, }; use city_store::config::F; use serde::{de::DeserializeOwned, Serialize}; +pub fn get_dummy_tree_prover_ids_op_circuit( + circuit_type: ProvingJobCircuitType, + dummy_type: ProvingJobCircuitType, + checkpoint_id: u64, + leaf_count: usize, +) -> Vec> { + let dummy_id = QProvingJobDataID::new_proof_job_id(checkpoint_id, dummy_type, 0xDD, 0, 0); + let leaves = (0..leaf_count) + .map(|i| QProvingJobDataID::core_op_witness(circuit_type, checkpoint_id, i as u32)) + .collect::>(); + get_dummy_tree_prover_ids(&leaves, dummy_id) +} + +pub fn get_dummy_tree_prover_ids_leaf_template( + leaf_template: QProvingJobDataID, + dummy_id: QProvingJobDataID, + leaf_count: usize, +) -> Vec> { + let leaves = (0..leaf_count) + .map(|i| leaf_template.with_task_index(i as u32)) + .collect::>(); + get_dummy_tree_prover_ids(&leaves, dummy_id) +} +pub fn get_dummy_tree_prover_ids( + leaves: &[QProvingJobDataID], + dummy_id: QProvingJobDataID, +) -> Vec> { + if leaves.len() == 0 { + vec![vec![dummy_id]] + } else { + let leaves_len = leaves.len(); + let levels = BinaryTreePlanner::new(leaves_len).levels; + let mut job_ids = vec![leaves.to_vec()]; + + for level_nodes in levels.into_iter() { + let mut level_job_ids: Vec = Vec::with_capacity(level_nodes.len()); + for node in level_nodes.into_iter() { + let left_proof_id = job_ids[node.left_job.level as usize] + [node.left_job.index as usize] + .get_output_id(); + let self_witness_id = left_proof_id.get_tree_parent_proof_input_id(); + level_job_ids.push(self_witness_id); + } + job_ids.push(level_job_ids); + } + job_ids + } +} pub fn plan_tree_prover_from_leaves< PS: QProofStore, LA: TPLeafAggregator, IO>, diff --git a/city_rollup_core_orchestrator/src/debug/scenario/mod.rs b/city_rollup_core_orchestrator/src/debug/scenario/mod.rs index 9aa34071..90554a6f 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/mod.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/mod.rs @@ -3,5 +3,4 @@ pub mod block_planner; pub mod process_requests; pub mod sc1; pub mod sc_plan; -pub mod sighash; -pub mod wallet; +pub mod sighash; \ No newline at end of file diff --git a/city_rollup_core_orchestrator/src/debug/scenario/process_requests/block_processor.rs b/city_rollup_core_orchestrator/src/debug/scenario/process_requests/block_processor.rs index c063f40f..d3092fc6 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/process_requests/block_processor.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/process_requests/block_processor.rs @@ -10,7 +10,7 @@ use city_rollup_common::{ }, qworker::{ fingerprints::CRWorkerToolboxCoreCircuitFingerprints, - job_id::{ProvingJobCircuitType, ProvingJobDataType, QJobTopic, QProvingJobDataID}, + job_id::{ProvingJobCircuitType, QProvingJobDataID}, job_witnesses::op::{ CRAddL1DepositCircuitInput, CRAddL1WithdrawalCircuitInput, CRClaimL1DepositCircuitInput, CRL2TransferCircuitInput, @@ -67,15 +67,10 @@ impl CityOrchestratorBlockProcessor { .op_processor .process_register_user_request(store, req)?; - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::RegisterUser, self.checkpoint_id, - ProvingJobCircuitType::RegisterUser.to_circuit_group_id(), - 0, self.block_register_user_count as u32, - ProvingJobCircuitType::RegisterUser, - ProvingJobDataType::InputWitness, - 0, ); proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; @@ -90,16 +85,10 @@ impl CityOrchestratorBlockProcessor { req: &CityTokenTransferRequest, ) -> anyhow::Result>> { let op_result = self.op_processor.process_l2_transfer_request(store, req)?; - - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::TransferTokensL2, self.checkpoint_id, - ProvingJobCircuitType::TransferTokensL2.to_circuit_group_id(), - 0, self.block_l2_transfer_count as u32, - ProvingJobCircuitType::TransferTokensL2, - ProvingJobDataType::InputWitness, - 0, ); proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; @@ -117,15 +106,11 @@ impl CityOrchestratorBlockProcessor { .op_processor .process_add_withdrawal_request(store, req)?; - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::AddL1Withdrawal, self.checkpoint_id, - ProvingJobCircuitType::AddL1Withdrawal.to_circuit_group_id(), - 0, self.block_add_withdrawal_count as u32, - ProvingJobCircuitType::AddL1Withdrawal, - ProvingJobDataType::InputWitness, - 0, ); proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; @@ -143,15 +128,11 @@ impl CityOrchestratorBlockProcessor { .op_processor .process_complete_l1_withdrawal_request(store, req)?; - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::ProcessL1Withdrawal, self.checkpoint_id, - ProvingJobCircuitType::ProcessL1Withdrawal.to_circuit_group_id(), - 0, self.block_process_withdrawal_count as u32, - ProvingJobCircuitType::ProcessL1Withdrawal, - ProvingJobDataType::InputWitness, - 0, ); proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; @@ -167,15 +148,10 @@ impl CityOrchestratorBlockProcessor { ) -> anyhow::Result>> { let op_result = self.op_processor.process_add_deposit_request(store, req)?; - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::AddL1Deposit, self.checkpoint_id, - ProvingJobCircuitType::AddL1Deposit.to_circuit_group_id(), - 0, self.block_add_deposit_count as u32, - ProvingJobCircuitType::AddL1Deposit, - ProvingJobDataType::InputWitness, - 0, ); proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; @@ -193,16 +169,12 @@ impl CityOrchestratorBlockProcessor { .op_processor .process_claim_deposit_request(store, req)?; - let job_id = QProvingJobDataID::new( - QJobTopic::GenerateStandardProof, + let job_id = QProvingJobDataID::core_op_witness( + ProvingJobCircuitType::ClaimL1Deposit, self.checkpoint_id, - ProvingJobCircuitType::ClaimL1Deposit.to_circuit_group_id(), - 0, self.block_claim_deposit_count as u32, - ProvingJobCircuitType::ClaimL1Deposit, - ProvingJobDataType::InputWitness, - 0, ); + proof_store.set_bytes_by_id(job_id, &op_result.to_bytes()?)?; self.block_claim_deposit_count += 1; diff --git a/city_rollup_core_orchestrator/src/debug/scenario/sc1.rs b/city_rollup_core_orchestrator/src/debug/scenario/sc1.rs index 48cb06a6..b62bc041 100644 --- a/city_rollup_core_orchestrator/src/debug/scenario/sc1.rs +++ b/city_rollup_core_orchestrator/src/debug/scenario/sc1.rs @@ -1,13 +1,13 @@ +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use kvq::traits::KVQBinaryStore; use plonky2::plonk::config::{AlgebraicHasher, GenericConfig}; -use super::wallet::DebugScenarioWallet; #[derive(Clone)] pub struct DebugScenarioBuilder + 'static, const D: usize> where C::Hasher: AlgebraicHasher, { - pub wallet: DebugScenarioWallet, + pub wallet: CityMemoryWallet, pub store: S, } diff --git a/city_rollup_core_orchestrator/src/debug/scenario/wallet.rs b/city_rollup_core_orchestrator/src/debug/scenario/wallet.rs deleted file mode 100644 index 744063bd..00000000 --- a/city_rollup_core_orchestrator/src/debug/scenario/wallet.rs +++ /dev/null @@ -1,208 +0,0 @@ -use city_common::{ - binaryhelpers::bytes::CompressedPublicKey, - config::rollup_constants::{DEPOSIT_FEE_AMOUNT, WITHDRAWAL_FEE_AMOUNT}, -}; -use city_common_circuit::{ - circuits::l1_secp256k1_signature::L1Secp256K1SignatureCircuit, - wallet::zk::{MemoryZKSignatureWallet, SimpleZKSignatureWallet, ZKSignatureWalletProvider}, -}; -use city_crypto::{ - hash::{ - base_types::{hash160::Hash160, hash256::Hash256}, - qhashout::QHashOut, - }, - signature::secp256k1::{ - core::QEDCompressedSecp256K1Signature, - wallet::{MemorySecp256K1Wallet, Secp256K1WalletProvider}, - }, -}; -use city_rollup_common::{ - api::data::{ - block::rpc_request::{ - CityAddWithdrawalRPCRequest, CityClaimDepositRPCRequest, CityTokenTransferRPCRequest, - }, - store::CityL1Deposit, - }, - introspection::rollup::signature::QEDSigAction, -}; -use plonky2::{ - hash::poseidon::PoseidonHash, - plonk::{ - config::{AlgebraicHasher, GenericConfig}, - proof::ProofWithPublicInputs, - }, -}; - -#[derive(Clone)] -pub struct DebugScenarioWallet + 'static, const D: usize> -where - C::Hasher: AlgebraicHasher, -{ - pub zk_wallet: MemoryZKSignatureWallet>, - pub secp256k1_circuit: Option>, - pub secp256k1_wallet: MemorySecp256K1Wallet, -} - -impl + 'static, const D: usize> DebugScenarioWallet -where - C::Hasher: AlgebraicHasher, -{ - pub fn new() -> Self { - Self { - zk_wallet: MemoryZKSignatureWallet::>::new_memory(), - secp256k1_wallet: MemorySecp256K1Wallet::new(), - secp256k1_circuit: Some(L1Secp256K1SignatureCircuit::new()), - } - } - pub fn new_fast_setup() -> Self { - Self { - zk_wallet: MemoryZKSignatureWallet::>::new_memory(), - secp256k1_wallet: MemorySecp256K1Wallet::new(), - secp256k1_circuit: None, - } - } - pub fn setup_circuits(&mut self) { - if self.secp256k1_circuit.is_none() { - self.secp256k1_circuit = Some(L1Secp256K1SignatureCircuit::new()); - } - } - pub fn add_zk_private_key(&mut self, private_key: QHashOut) -> QHashOut { - self.zk_wallet.add_private_key(private_key) - } - pub fn add_secp256k1_private_key( - &mut self, - private_key: Hash256, - ) -> anyhow::Result { - self.secp256k1_wallet.add_private_key(private_key) - } - pub fn zk_sign( - &self, - fingerprint_hash: QHashOut, - message: QHashOut, - ) -> anyhow::Result> { - self.zk_wallet.sign(fingerprint_hash, message) - } - pub fn zk_sign_hash_public_key( - &self, - hash_public_key: QHashOut, - message: QHashOut, - ) -> anyhow::Result> { - self.zk_wallet - .sign_by_hash_public_key(hash_public_key, message) - } - pub fn sign_secp256k1( - &self, - public_key: CompressedPublicKey, - message: Hash256, - ) -> anyhow::Result { - self.secp256k1_wallet.sign(&public_key, message) - } - pub fn sign_hash_secp256k1( - &self, - public_key: CompressedPublicKey, - message: QHashOut, - ) -> anyhow::Result { - self.secp256k1_wallet.sign_qhashout(&public_key, message) - } - pub fn zk_sign_secp256k1( - &self, - public_key: CompressedPublicKey, - message: Hash256, - ) -> anyhow::Result> { - let ecc_sig = self.sign_secp256k1(public_key, message)?; - self.secp256k1_circuit.as_ref().unwrap().prove(&ecc_sig) - } - pub fn zk_sign_hash_secp256k1( - &self, - public_key: CompressedPublicKey, - message: QHashOut, - ) -> anyhow::Result> { - let ecc_sig = self.sign_hash_secp256k1(public_key, message)?; - self.secp256k1_circuit.as_ref().unwrap().prove(&ecc_sig) - } - pub fn sign_claim_deposit( - &self, - network_magic: u64, - user_id: u64, - l1_deposit: &CityL1Deposit, - ) -> anyhow::Result { - let sig_preimage = QEDSigAction::::new_claim_deposit_action( - network_magic, - user_id, - l1_deposit.txid, - l1_deposit.value, - DEPOSIT_FEE_AMOUNT, - ); - tracing::info!( - "sig_preimage: {}", - serde_json::to_string(&sig_preimage).unwrap() - ); - let hash = sig_preimage.get_qhash::(); - let proof = self.zk_sign_hash_secp256k1(l1_deposit.public_key, hash)?; - - let signature_proof = bincode::serialize(&proof)?; - Ok(CityClaimDepositRPCRequest { - public_key: l1_deposit.public_key.0, - user_id, - deposit_id: l1_deposit.deposit_id, - value: l1_deposit.value, - txid: l1_deposit.txid, - signature_proof, - }) - } - pub fn sign_l2_transfer( - &self, - public_key: QHashOut, - network_magic: u64, - from: u64, - to: u64, - value: u64, - nonce: u64, - ) -> anyhow::Result { - let sig_preimage = - QEDSigAction::::new_transfer_action(network_magic, from, nonce, to, value); - let hash = sig_preimage.get_qhash::(); - let proof = self.zk_sign(public_key, hash)?; - - let signature_proof = bincode::serialize(&proof)?; - Ok(CityTokenTransferRPCRequest { - signature_proof, - to, - nonce, - user_id: from, - value: value, - }) - } - pub fn sign_withdrawal( - &self, - public_key: QHashOut, - network_magic: u64, - user_id: u64, - l1_address: Hash160, - value: u64, - nonce: u64, - ) -> anyhow::Result { - let sig_preimage: QEDSigAction = - QEDSigAction::::new_withdrawal_action::( - network_magic, - user_id, - nonce, - l1_address, - 0, - value, - WITHDRAWAL_FEE_AMOUNT, - ); - let hash = sig_preimage.get_qhash::(); - let proof = self.zk_sign(public_key, hash)?; - - let signature_proof = bincode::serialize(&proof)?; - Ok(CityAddWithdrawalRPCRequest { - signature_proof, - nonce, - user_id: user_id, - value: value, - destination_type: 0, - destination: l1_address, - }) - } -} diff --git a/city_rollup_core_orchestrator/src/event_receiver.rs b/city_rollup_core_orchestrator/src/event_receiver.rs index 84786576..3214c10f 100644 --- a/city_rollup_core_orchestrator/src/event_receiver.rs +++ b/city_rollup_core_orchestrator/src/event_receiver.rs @@ -22,6 +22,7 @@ use city_rollup_worker_dispatch::traits::proving_worker::ProvingWorkerListener; use plonky2::hash::hash_types::RichField; use serde::de::DeserializeOwned; +#[derive(Clone)] pub struct CityEventReceiver { tx_queue: RedisQueue, rpc_processor: QRPCProcessor, diff --git a/city_rollup_core_orchestrator/src/lib.rs b/city_rollup_core_orchestrator/src/lib.rs index a3be1a24..9d11f7b1 100644 --- a/city_rollup_core_orchestrator/src/lib.rs +++ b/city_rollup_core_orchestrator/src/lib.rs @@ -1,9 +1,10 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use city_common::{cli::args::OrchestratorArgs, units::UNIT_BTC}; use city_crypto::hash::{base_types::hash256::Hash256, qhashout::QHashOut}; use city_macros::sync_infinite_loop; use city_redis_store::RedisStore; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_common::{ actors::{ rpc_processor::QRPCProcessor, @@ -17,17 +18,18 @@ use city_rollup_common::{ data::BTCAddress160, link_api::BTCLinkAPI, traits::QBitcoinAPIFunderSync, tx::setup_genesis_block, }, - qworker::fingerprints::CRWorkerToolboxCoreCircuitFingerprints, + qworker::{fingerprints::CRWorkerToolboxCoreCircuitFingerprints, proof_store::QDummyProofStore}, }; +use city_rollup_core_api::KV; use city_rollup_core_worker::event_processor::CityEventProcessor; use city_rollup_worker_dispatch::implementations::redis::RedisQueue; use city_store::store::{city::base::CityStore, sighash::SigHashMerkleTree}; -use kvq_store_rocksdb::KVQRocksDBStore; +use kvq_store_redb::KVQReDBStore; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; +use redb::Database; use crate::{ - debug::scenario::{actors::simple::SimpleActorOrchestrator, wallet::DebugScenarioWallet}, - event_receiver::CityEventReceiver, + debug::scenario::actors::simple::SimpleActorOrchestrator, event_receiver::CityEventReceiver, }; pub mod debug; @@ -42,11 +44,10 @@ pub fn run(args: OrchestratorArgs) -> anyhow::Result<()> { let queue = RedisQueue::new(&args.redis_uri)?; let mut event_processor = CityEventProcessor::new(queue.clone()); let fingerprints: CRWorkerToolboxCoreCircuitFingerprints = serde_json::from_str( - /* + /* r#" {"network_magic":1384803358401167209,"zk_signature_wrapper":"2efad90d446638deb0af8cdc8efec541a82ee5ab2b6d221bd7d57af5885fe480","l1_secp256k1_signature":"0e06b3318325a6e4b2611b75767a366f79d50c039967f13760fe106d8560735b","op_register_user":{"leaf_fingerprint":"3b3c690b289d78d2e2acd9678d919f34534cbb1946a4edab39687951b2d8df3b","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"1860a3680b473aaea4f1a26f855890bb325fee1f1019b5a160483fd4f30294f8","leaf_circuit_type":0,"aggregator_circuit_type":1},"op_claim_l1_deposit":{"leaf_fingerprint":"e21b3c53d7f942bfca301943124794dd8ccefd6093ea490669ffd19ca26c0226","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"7fa5739771d7275eb11230444803a58f62918509b42db403b9670f3adc9fc9cc","leaf_circuit_type":4,"aggregator_circuit_type":5},"op_l2_transfer":{"leaf_fingerprint":"4e48381c9e08a9b592a088fa03dfd9e7935af7a0636ea420901b0a28cb9c55df","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"f47cf0cc794240b402d3356facfcdc20dafc5b0769de4cd24ec7ac464c12f34a","leaf_circuit_type":6,"aggregator_circuit_type":7},"op_add_l1_withdrawal":{"leaf_fingerprint":"db3e0e786e52f327d16d1b4329694cebc908674e72a4a92a2746f64b279f05e4","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"f353c1583cf9de84e8344b3b58d496b92c34016929f36a23bcc556fe434e3f59","leaf_circuit_type":8,"aggregator_circuit_type":9},"op_add_l1_deposit":{"leaf_fingerprint":"9cbbe2dd4a47b04a15441ccbfe95264130c22d6387cc9cab15c50c2fbeb6b3a8","aggregator_fingerprint":"a97d6231eaba54ccd65185134b7e830562540d933598d9decc6ec36bf4f632d5","dummy_fingerprint":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca","allowed_circuit_hashes_root":"b4a4c9b5f8c9af6e9c76946bb3aff7d6f1471061d67411b07cd4bd3da392fcff","leaf_circuit_type":2,"aggregator_circuit_type":3},"op_process_l1_withdrawal":{"leaf_fingerprint":"9aca81a13566a4529ef78c4385e9e6dddd157f54aef7b23d9501f1ea98541e03","aggregator_fingerprint":"a97d6231eaba54ccd65185134b7e830562540d933598d9decc6ec36bf4f632d5","dummy_fingerprint":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca","allowed_circuit_hashes_root":"1ef92f20ea565bbfe49d75997e82b965c0887714bb98065a2463b72f555b060b","leaf_circuit_type":10,"aggregator_circuit_type":11},"agg_state_transition":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","agg_state_transition_with_events":"a97d6231eaba54ccd65185134b7e830562540d933598d9decc6ec36bf4f632d5","agg_state_transition_dummy":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","agg_state_transition_with_events_dummy":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca"} "#,*/ - r#" {"network_magic":1384803358401167209,"zk_signature_wrapper":"2efad90d446638deb0af8cdc8efec541a82ee5ab2b6d221bd7d57af5885fe480","l1_secp256k1_signature":"0e06b3318325a6e4b2611b75767a366f79d50c039967f13760fe106d8560735b","op_register_user":{"leaf_fingerprint":"3b3c690b289d78d2e2acd9678d919f34534cbb1946a4edab39687951b2d8df3b","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"1860a3680b473aaea4f1a26f855890bb325fee1f1019b5a160483fd4f30294f8","leaf_circuit_type":0,"aggregator_circuit_type":1},"op_claim_l1_deposit":{"leaf_fingerprint":"a97c868fcd025a2763b6c03581729d0108a709eddfdadf209c3eef99a160a50f","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"b397fee16231a678ef08fb1bd7fd4cbca63a12d6ef0d2586a2b5f1dc3cc5b74b","leaf_circuit_type":4,"aggregator_circuit_type":5},"op_l2_transfer":{"leaf_fingerprint":"6e7817a58684785bb726c1c04ed544870b1d86c4b907815ed07694a65a76ad93","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"1c88193a6cde038e1120a42260a015c5247f3710e232aba8306f960cc55e33f2","leaf_circuit_type":6,"aggregator_circuit_type":7},"op_add_l1_withdrawal":{"leaf_fingerprint":"794761e0ceaf2a20be43877eced9db4c938b426c89785cf9d3f4773556086c84","aggregator_fingerprint":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","dummy_fingerprint":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","allowed_circuit_hashes_root":"cf222a8fa3c1f30f7266ebad8b0bd54c92029a0884ff151c26f8b08af790ff8f","leaf_circuit_type":8,"aggregator_circuit_type":9},"op_add_l1_deposit":{"leaf_fingerprint":"9cbbe2dd4a47b04a15441ccbfe95264130c22d6387cc9cab15c50c2fbeb6b3a8","aggregator_fingerprint":"6133fd6b95240863dc4458e6a6721a2bb37ea8f81080086ed775a32589a85f34","dummy_fingerprint":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca","allowed_circuit_hashes_root":"f4f90b1affb54b9bc0c110eab2276b943b4e352551c2c31e888d3e93be0a1858","leaf_circuit_type":2,"aggregator_circuit_type":3},"op_process_l1_withdrawal":{"leaf_fingerprint":"9aca81a13566a4529ef78c4385e9e6dddd157f54aef7b23d9501f1ea98541e03","aggregator_fingerprint":"6133fd6b95240863dc4458e6a6721a2bb37ea8f81080086ed775a32589a85f34","dummy_fingerprint":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca","allowed_circuit_hashes_root":"8c43a100d8e93a1dfdb0c5bb4830b31c4cb948f4e7d84fcc0419798ac331a90f","leaf_circuit_type":10,"aggregator_circuit_type":11},"agg_state_transition":"6d1911dc4660dc9b2e61a581a5c1608b7ef97c2971e7117e8e121be2dc362dce","agg_state_transition_with_events":"6133fd6b95240863dc4458e6a6721a2bb37ea8f81080086ed775a32589a85f34","agg_state_transition_dummy":"1a408fbe18d03c1c7886cc7f1906a07989535d2a995d4b16eacaa4c739df628b","agg_state_transition_with_events_dummy":"081162f1ae48232a6d4a1e9c35adc0b4f2349fcaa740fa6034a7542e0ed1e5ca"} "# )?; @@ -54,7 +55,7 @@ pub fn run(args: OrchestratorArgs) -> anyhow::Result<()> { let mut rpc_queue = CityEventReceiver::::new(queue.clone(), QRPCProcessor::new(0), proof_store.clone()); - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); let genesis_funder_public_key = wallet.add_secp256k1_private_key(Hash256( hex_literal::hex!("133700f4676a0d0e16aaced646ed693626fcf1329db55be8eee13ad8df001337"), ))?; @@ -74,57 +75,71 @@ pub fn run(args: OrchestratorArgs) -> anyhow::Result<()> { checkpoint_id: 1, ..Default::default() }; - let mut store = KVQRocksDBStore::open_default(&args.db_path)?; - let storec = store.clone(); - std::thread::spawn(move || { - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); + let db = Arc::new(Database::create(&args.db_path)?); + let wxn = db.begin_write()?; + { + let table = wxn.open_table(KV)?; + let mut store = KVQReDBStore::new(table); + let expose_proof_store_api = args.expose_proof_store_api; + let api_proof_store = proof_store.clone(); - let _ = rt.block_on(async move { - tracing::info!("api server listening on http://{}", args.server_addr); - city_rollup_core_api::run_server(args.server_addr, storec).await?; - Ok::<_, anyhow::Error>(()) - }); - }); - CityStore::set_block_state(&mut store, &block0)?; - CityStore::set_block_state(&mut store, &block1)?; + let dbc = db.clone(); + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); - let genesis_state_hash = CityStore::get_city_root(&store, 0)?; - let setup_fee = 100000 * 500; - let fund_genesis_txid = api.fund_address_from_random_p2pkh_address( - genesis_funder_address, - 101 * UNIT_BTC + setup_fee * 4, - )?; + let _ = rt.block_on(async move { + tracing::info!("api server listening on http://{}", args.server_addr); + if expose_proof_store_api { + city_rollup_core_api::run_server(args.server_addr, dbc, api_proof_store).await?; + } else { + city_rollup_core_api::run_server(args.server_addr, dbc, QDummyProofStore::new()).await?; + } + Ok::<_, anyhow::Error>(()) + }); + }); + CityStore::set_block_state(&mut store, &block0)?; + CityStore::set_block_state(&mut store, &block1)?; - api.mine_blocks(1)?; - let txid_fund_genesis = setup_genesis_block( - &api, - &wallet.secp256k1_wallet, - genesis_funder_address.address, - fund_genesis_txid, - setup_fee, - genesis_state_hash.to_felt248_hash256(), - )?; - tracing::info!( - "funded genesis block with txid: {}", - txid_fund_genesis.to_hex_string() - ); - let _block_2_address = - BTCAddress160::new_p2sh(CityStore::get_city_block_deposit_address(&store, 2)?); - api.mine_blocks(1)?; - let user_0_public_key = wallet.add_zk_private_key(QHashOut::from_values(100, 100, 100, 100)); - let user_1_public_key = wallet.add_zk_private_key(QHashOut::from_values(101, 101, 101, 101)); - let _ = wallet.add_zk_private_key(QHashOut::from_values(102, 102, 102, 102)); - let _ = wallet.add_zk_private_key(QHashOut::from_values(103, 103, 103, 103)); - let register_user_rpc_events = - CityRegisterUserRPCRequest::new_batch(&[user_0_public_key, user_1_public_key]); - let _ = register_user_rpc_events - .into_iter() - .map(|x| rpc_queue.notify_rpc_register_user(&x)) - .collect::>>()?; + let genesis_state_hash = CityStore::get_city_root(&store, 0)?; + let setup_fee = 100000 * 500; + let fund_genesis_txid = api.fund_address_from_random_p2pkh_address( + genesis_funder_address, + 101 * UNIT_BTC + setup_fee * 4, + )?; + api.mine_blocks(1)?; + let txid_fund_genesis = setup_genesis_block( + &api, + &wallet.secp256k1_wallet, + genesis_funder_address.address, + fund_genesis_txid, + setup_fee, + genesis_state_hash.to_felt248_hash256(), + )?; + tracing::info!( + "funded genesis block with txid: {}", + txid_fund_genesis.to_hex_string() + ); + let _block_2_address = + BTCAddress160::new_p2sh(CityStore::get_city_block_deposit_address(&store, 2)?); + api.mine_blocks(1)?; + let user_0_public_key = + wallet.add_zk_private_key(QHashOut::from_values(100, 100, 100, 100)); + let user_1_public_key = + wallet.add_zk_private_key(QHashOut::from_values(101, 101, 101, 101)); + let _ = wallet.add_zk_private_key(QHashOut::from_values(102, 102, 102, 102)); + let _ = wallet.add_zk_private_key(QHashOut::from_values(103, 103, 103, 103)); + let register_user_rpc_events = + CityRegisterUserRPCRequest::new_batch(&[user_0_public_key, user_1_public_key]); + let _ = register_user_rpc_events + .into_iter() + .map(|x| rpc_queue.notify_rpc_register_user(&x)) + .collect::>>()?; + } + wxn.commit()?; /* let user_0_public_key = wallet.add_zk_private_key(QHashOut::from_values(100, 100, 100, 100)); @@ -157,35 +172,41 @@ pub fn run(args: OrchestratorArgs) -> anyhow::Result<()> { */ sync_infinite_loop!(1000, { - let block_state = CityStore::get_latest_block_state(&store)?; - tracing::info!( - "last_block_state.checkpoint_id: {}", - block_state.checkpoint_id - ); - let mut event_receiver = CityEventReceiver::::new( - queue.clone(), - QRPCProcessor::new(block_state.checkpoint_id + 1), - proof_store.clone(), - ); - event_receiver.wait_for_produce_block()?; - let orchestrator_result_step_1 = - SimpleActorOrchestrator::step_1_produce_block_enqueue_jobs( + let wxn = db.begin_write()?; + { + let table = wxn.open_table(KV)?; + let mut store = KVQReDBStore::new(table); + let block_state = CityStore::get_latest_block_state(&store)?; + tracing::info!( + "last_block_state.checkpoint_id: {}", + block_state.checkpoint_id + ); + let mut event_receiver = CityEventReceiver::::new( + queue.clone(), + QRPCProcessor::new(block_state.checkpoint_id + 1), + proof_store.clone(), + ); + event_receiver.wait_for_produce_block()?; + let orchestrator_result_step_1 = + SimpleActorOrchestrator::step_1_produce_block_enqueue_jobs( + &mut proof_store, + &mut store, + &mut event_receiver, + &mut event_processor, + &mut api, + &fingerprints, + &sighash_whitelist_tree, + )?; + event_processor.wait_for_block_proving_jobs(block_state.checkpoint_id + 1)?; + api.mine_blocks(1)?; + let txid = SimpleActorOrchestrator::step_2_produce_block_finalize_and_transact( &mut proof_store, - &mut store, - &mut event_receiver, - &mut event_processor, &mut api, - &fingerprints, - &sighash_whitelist_tree, + &orchestrator_result_step_1, )?; - event_processor.wait_for_block_proving_jobs(block_state.checkpoint_id + 1)?; - api.mine_blocks(1)?; - let txid = SimpleActorOrchestrator::step_2_produce_block_finalize_and_transact( - &mut proof_store, - &mut api, - &orchestrator_result_step_1, - )?; - tracing::info!("funded next block: {}",txid.to_hex_string()); - api.mine_blocks(1)?; + tracing::info!("funded next block: {}", txid.to_hex_string()); + api.mine_blocks(1)?; + } + wxn.commit()?; }); } diff --git a/city_rollup_core_worker/src/actors/simple.rs b/city_rollup_core_worker/src/actors/simple.rs index 53f0dc64..b17cd59f 100644 --- a/city_rollup_core_worker/src/actors/simple.rs +++ b/city_rollup_core_worker/src/actors/simple.rs @@ -1,7 +1,7 @@ use std::time::Duration; use city_common::{cli::modes::QWorkerMode, logging::trace_timer::TraceTimer}; -use city_rollup_circuit::worker::traits::{QWorkerGenericProver, QWorkerGenericProverGroth16}; +use city_rollup_circuit::worker::traits::{QWorkerGenericProverGroth16, QWorkerGenericProverMut}; use city_rollup_common::{ actors::traits::WorkerEventReceiverSync, qworker::{ @@ -16,14 +16,14 @@ impl SimpleActorWorker { pub fn run_worker< PS: QProofStore, ER: WorkerEventReceiverSync, - G: QWorkerGenericProver + G: QWorkerGenericProverMut + QWorkerGenericProverGroth16, C: GenericConfig, const D: usize, >( store: &mut PS, event_receiver: &mut ER, - prover: &G, + prover: &mut G, ) -> anyhow::Result<()> { loop { Self::process_next_job(store, event_receiver, prover, QWorkerMode::All)?; @@ -32,14 +32,14 @@ impl SimpleActorWorker { pub fn process_next_job< PS: QProofStore, ER: WorkerEventReceiverSync, - G: QWorkerGenericProver + G: QWorkerGenericProverMut + QWorkerGenericProverGroth16, C: GenericConfig, const D: usize, >( store: &mut PS, event_receiver: &mut ER, - prover: &G, + prover: &mut G, mode: QWorkerMode, ) -> anyhow::Result<()> { //let mut timer = TraceTimer::new("process_next_job"); @@ -57,18 +57,19 @@ impl SimpleActorWorker { fn process_job< PS: QProofStore, ER: WorkerEventReceiverSync, - G: QWorkerGenericProver + G: QWorkerGenericProverMut + QWorkerGenericProverGroth16, C: GenericConfig, const D: usize, >( store: &mut PS, event_receiver: &mut ER, - prover: &G, + prover: &mut G, job_id: QProvingJobDataID, ) -> anyhow::Result<()> { let mut timer = TraceTimer::new("process_job"); if job_id.topic == QJobTopic::GenerateStandardProof { + let start_time = std::time::Instant::now(); let _ = match job_id.circuit_type { ProvingJobCircuitType::WrapFinalSigHashProofBLS12381 => { // TODO: implement conversion from proof to bytes @@ -79,12 +80,14 @@ impl SimpleActorWorker { output_id } _ => { - let proof = prover.worker_prove(store, job_id)?; + let proof = prover.worker_prove_mut(store, job_id)?; let output_id = job_id.get_output_id(); store.set_proof_by_id(output_id, &proof)?; output_id } }; + let duration = start_time.elapsed().as_millis() as u64; + event_receiver.record_job_bench(job_id, duration)?; } if job_id.topic == QJobTopic::NotifyOrchestratorComplete { event_receiver.notify_core_goal_completed(job_id)?; diff --git a/city_rollup_core_worker/src/event_processor.rs b/city_rollup_core_worker/src/event_processor.rs index 0ca95750..14c8f20f 100644 --- a/city_rollup_core_worker/src/event_processor.rs +++ b/city_rollup_core_worker/src/event_processor.rs @@ -2,20 +2,27 @@ use std::time::Duration; use city_rollup_common::{ actors::traits::{WorkerEventReceiverSync, WorkerEventTransmitterSync}, - qworker::job_id::QProvingJobDataID, + qworker::job_id::{QProvingJobDataID, QWorkerJobBenchmark}, }; use city_rollup_worker_dispatch::{ implementations::redis::{QueueNotification, RedisQueue, Q_JOB, Q_NOTIFICATIONS}, traits::{proving_dispatcher::ProvingDispatcher, proving_worker::ProvingWorkerListener}, }; - +#[derive(Clone)] pub struct CityEventProcessor { pub job_queue: RedisQueue, + pub benckmarks_enabled: bool, + pub benchmarks: Vec, } impl CityEventProcessor { pub fn new(dispatcher: RedisQueue) -> Self { + Self::new_with_config(dispatcher, false) + } + pub fn new_with_config(dispatcher: RedisQueue, benckmarks_enabled: bool) -> Self { Self { job_queue: dispatcher, + benckmarks_enabled, + benchmarks: Vec::new(), } } } @@ -43,6 +50,16 @@ impl WorkerEventReceiverSync for CityEventProcessor { self.job_queue.dispatch(Q_NOTIFICATIONS, QueueNotification::CoreJobCompleted)?; Ok(()) } + + fn record_job_bench(&mut self, job: QProvingJobDataID, duration: u64) -> anyhow::Result<()> { + if self.benckmarks_enabled { + self.benchmarks.push(QWorkerJobBenchmark { + job_id: job.to_fixed_bytes(), + duration, + }); + } + Ok(()) + } } impl WorkerEventTransmitterSync for CityEventProcessor { diff --git a/city_rollup_core_worker/src/lib.rs b/city_rollup_core_worker/src/lib.rs index 6dd5ddea..21a5100b 100644 --- a/city_rollup_core_worker/src/lib.rs +++ b/city_rollup_core_worker/src/lib.rs @@ -26,7 +26,7 @@ const D: usize = 2; type C = PoseidonGoldilocksConfig; pub fn run_debug_outer(args: L2WorkerArgs) -> anyhow::Result<()> { let network_magic = get_network_magic_for_str(args.network.to_string())?; - let toolbox = + let mut toolbox = CRWorkerToolboxRootCircuits::::new(network_magic, SIGHASH_WHITELIST_TREE_ROOT); /*println!( "CRWorkerToolboxCoreCircuitFingerprints: {}", @@ -41,7 +41,7 @@ pub fn run_debug_outer(args: L2WorkerArgs) -> anyhow::Result<()> { } println!("worker ready"); loop { - run_debug_inner(&args, &toolbox)?; + run_debug_inner(&args, &mut toolbox)?; println!("press enter to reset worker"); enable_raw_mode()?; loop { @@ -59,12 +59,13 @@ pub fn run_debug_outer(args: L2WorkerArgs) -> anyhow::Result<()> { } pub fn run_debug_inner( args: &L2WorkerArgs, - toolbox: &CRWorkerToolboxRootCircuits, + toolbox: &mut CRWorkerToolboxRootCircuits, ) -> anyhow::Result<()> { let job_queue = RedisQueue::new(&args.redis_uri)?; let mut proof_store = RedisStore::new(&args.redis_uri)?; - let mut event_processor = CityEventProcessor::new(job_queue.clone()); + let mut event_processor = CityEventProcessor::new_with_config(job_queue.clone(), true); + let mut should_print_benchmark = false; loop { 'inner: loop { if event_processor.job_queue.is_empty() { @@ -83,6 +84,10 @@ pub fn run_debug_inner( if event == Event::Key(KeyCode::Esc.into()) { break; } + if event == Event::Key(KeyCode::Char('p').into()) { + should_print_benchmark = true; + break; + } } disable_raw_mode()?; @@ -90,6 +95,10 @@ pub fn run_debug_inner( } disable_raw_mode()?; + + if should_print_benchmark { + println!("benchmarks: {}", serde_json::to_string(&event_processor.benchmarks)?); + } Ok(()) } pub fn run(args: L2WorkerArgs) -> anyhow::Result<()> { @@ -101,7 +110,7 @@ pub fn run(args: L2WorkerArgs) -> anyhow::Result<()> { let network_magic = get_network_magic_for_str(args.network.to_string())?; let mut event_processor = CityEventProcessor::new(job_queue.clone()); - let toolbox = + let mut toolbox = CRWorkerToolboxRootCircuits::::new(network_magic, SIGHASH_WHITELIST_TREE_ROOT); //println!("fingerprints:\n{}", serde_json::to_string(&toolbox.core.fingerprints).unwrap()); @@ -127,7 +136,7 @@ pub fn run(args: L2WorkerArgs) -> anyhow::Result<()> { SimpleActorWorker::process_next_job( &mut proof_store, &mut event_processor, - &toolbox, + &mut toolbox, args.worker_mode, )?; } diff --git a/city_rollup_core_worker_qbench/Cargo.toml b/city_rollup_core_worker_qbench/Cargo.toml new file mode 100644 index 00000000..dee42388 --- /dev/null +++ b/city_rollup_core_worker_qbench/Cargo.toml @@ -0,0 +1,39 @@ +[package] +edition = "2021" +name = "city_rollup_core_worker_qbench" +version = "0.1.0" + + +[dependencies] +plonky2 = { workspace = true } +anyhow = { workspace = true } +serde_json = { workspace = true } +serde = { workspace = true } +serde_with = { workspace = true } +hex = { workspace = true } +rand = { workspace = true } +bitcoin = { workspace = true } +reqwest = { workspace = true } +bincode = { workspace = true } +url = { workspace = true } +k256 = { workspace = true } +city_crypto = { path = "../city_crypto" } +city_common_circuit = { path = "../city_common_circuit" } +city_rollup_core_worker = { path = "../city_rollup_core_worker" } +city_rollup_core_orchestrator = { path = "../city_rollup_core_orchestrator" } +city_common = { path = "../city_common" } +city_rollup_common = { path = "../city_rollup_common" } +tokio = { workspace = true } +city_macros = { path = "../city_macros" } +city_redis_store = { path = "../city_redis_store" } +city_rollup_circuit = { path = "../city_rollup_circuit" } +city_rollup_worker_dispatch = { path = "../city_rollup_worker_dispatch" } +city_store = { path = "../city_store" } +gnark-plonky2-wrapper = { workspace = true } +home = { workspace = true } +tracing = { workspace = true } +crossterm = "0.27.0" +[dev-dependencies] +criterion = "0.5.1" +rand_chacha = "0.3.1" +hex-literal = "0.4.1" diff --git a/city_rollup_core_worker_qbench/src/dump.rs b/city_rollup_core_worker_qbench/src/dump.rs new file mode 100644 index 00000000..6ac5f58d --- /dev/null +++ b/city_rollup_core_worker_qbench/src/dump.rs @@ -0,0 +1,161 @@ +use city_common::cli::args::L2DumpProofStoreArgs; +use city_redis_store::RedisStore; +use city_rollup_common::qworker::{ + dump::dump_job_dependencies_from_store, + job_id::{ProvingJobCircuitType, QProvingJobDataID}, + memory_proof_store::SimpleProofStoreMemory, + proof_store::{QProofStore, QProofStoreReaderSync}, +}; +use city_rollup_core_orchestrator::debug::scenario::{ + actors::job_planner::plan_jobs, + block_planner::transition::{CityOpJobConfig, CityOpJobIds}, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Copy, Clone)] +pub struct DumpProofStoreConfig { + pub checkpoint_id: u64, + pub rpc_node_id: u32, + pub job_config: CityOpJobConfig, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct BlockProofStoreDump { + pub config: DumpProofStoreConfig, + pub store: SimpleProofStoreMemory, +} +fn mirror_proof_store(keys: &[QProvingJobDataID], source: &PSource, destination: &mut PDestination) -> anyhow::Result<()> { + for key in keys { + let data = source.get_bytes_by_id(*key)?; + destination.set_bytes_by_id(*key, &data)?; + } + Ok(()) +} +// this only works when running a single rpc node with a the node id "rpc_node_id", which is ok because this is for debugging only +pub fn get_rpc_proof_dependencies( + config: &DumpProofStoreConfig, +) -> anyhow::Result> { + let token_transfer_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::transfer_signature_proof(config.rpc_node_id, config.checkpoint_id, i as u32) + }); + + let claim_deposit_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::claim_deposit_l1_signature_proof( + config.rpc_node_id, + config.checkpoint_id, + i as u32, + ) + }); + + let withdrawal_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::withdrawal_signature_proof(config.rpc_node_id, config.checkpoint_id, i as u32) + }); + + Ok(token_transfer_signature_proof_ids + .chain(claim_deposit_signature_proof_ids) + .chain(withdrawal_signature_proof_ids) + .collect()) +} +pub fn dump_proof_store( + config: &DumpProofStoreConfig, + real_store: &PS, +) -> anyhow::Result { + let mut mirror_store = SimpleProofStoreMemory::new(); + let block_op_job_ids = + CityOpJobIds::dummy_from_config(config.checkpoint_id, &config.job_config); + let num_input_witnesses = config.job_config.add_deposit_count + 1; + let leaves = plan_jobs( + &mut mirror_store, + &block_op_job_ids, + num_input_witnesses, + config.checkpoint_id, + )?; + + /* + println!( + "leaf_jobs: {}", + serde_json::to_string( + &leaves + .iter() + .map(|x| QProvingJobDataIDSerializedWrapped(x.to_fixed_bytes())) + .collect::>() + )? + ); + */ + + + let dependency_map = dump_job_dependencies_from_store(real_store, &leaves)?; + + let dep_tree = dependency_map.get_dependency_tree_for_block(config.checkpoint_id); + let rpc_signature_proof_dependencies = get_rpc_proof_dependencies(config)?; + mirror_proof_store(&rpc_signature_proof_dependencies, real_store, &mut mirror_store)?; + + + let proof_witnesses = dep_tree.get_all_dependencies(); + mirror_proof_store(&proof_witnesses, real_store, &mut mirror_store)?; + + + Ok(BlockProofStoreDump{ + store: mirror_store, + config: config.clone(), + }) +} + +pub fn get_leaf_count_or_dummy(store: &PS, circuit_type: ProvingJobCircuitType, dummy_type: ProvingJobCircuitType, checkpoint_id: u64) -> anyhow::Result { + let counter_job_id = QProvingJobDataID::core_op_witness(circuit_type, checkpoint_id, 0).get_sub_group_counter_goal_id(); + + let dummy_job_id = QProvingJobDataID::new_proof_job_id( + checkpoint_id, + dummy_type, + 0xDD, + 0, + 0, + ); + + let counter_bytes_result = store.get_bytes_by_id(counter_job_id).unwrap_or(vec![]); + if counter_bytes_result.len() == 4 { + let counter = u32::from_le_bytes(counter_bytes_result.try_into().unwrap()); + Ok(counter as usize) + }else{ + let dummy_bytes_result = store.get_bytes_by_id(dummy_job_id); + if dummy_bytes_result.is_ok() && dummy_bytes_result.unwrap().len() != 0 { + Ok(0) + }else{ + anyhow::bail!("no counter or dummy job found for circuit type {:?} and checkpoint_id {}", circuit_type, checkpoint_id) + } + } +} +pub fn get_proof_store_config(store: &PS, checkpoint_id: u64, rpc_node_id: u32) -> anyhow::Result { + let register_user_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::RegisterUser, ProvingJobCircuitType::DummyRegisterUserAggregate, checkpoint_id)?; + let add_deposit_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::AddL1Deposit, ProvingJobCircuitType::DummyAddL1DepositAggregate, checkpoint_id)?; + let token_transfer_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::TransferTokensL2, ProvingJobCircuitType::DummyTransferTokensL2Aggregate, checkpoint_id)?; + let add_withdrawal_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::AddL1Withdrawal, ProvingJobCircuitType::DummyAddL1WithdrawalAggregate, checkpoint_id)?; + let process_withdrawal_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::ProcessL1Withdrawal, ProvingJobCircuitType::DummyProcessL1WithdrawalAggregate, checkpoint_id)?; + let claim_deposit_count = get_leaf_count_or_dummy(store, ProvingJobCircuitType::ClaimL1Deposit, ProvingJobCircuitType::DummyClaimL1DepositAggregate, checkpoint_id)?; + let job_config = CityOpJobConfig { + register_user_count, + claim_deposit_count, + token_transfer_count, + add_withdrawal_count, + process_withdrawal_count, + add_deposit_count, + }; + + + + Ok(DumpProofStoreConfig { + checkpoint_id, + rpc_node_id, + job_config, + }) +} +pub fn run_dump_block_proof_store(args: &L2DumpProofStoreArgs) -> anyhow::Result<()>{ + let root = std::env::current_dir()?; + let output_path = root.join(args.output.clone()).display().to_string(); + let real_store = RedisStore::new(&args.redis_uri)?; + let config = get_proof_store_config(&real_store, args.checkpoint_id, 0)?; + //println!("got config: {}", serde_json::to_string_pretty(&config)?); + let result = dump_proof_store(&config, &real_store)?; + std::fs::write(output_path, &bincode::serialize(&result)?)?; + Ok(()) +} diff --git a/city_rollup_core_worker_qbench/src/inspect.rs b/city_rollup_core_worker_qbench/src/inspect.rs new file mode 100644 index 00000000..c2382934 --- /dev/null +++ b/city_rollup_core_worker_qbench/src/inspect.rs @@ -0,0 +1,137 @@ +use city_common::cli::{args::InspectL2DumpArgs, modes::QDumpInspectionData}; +use city_rollup_common::qworker::{ + dump::dump_job_dependencies_from_store, job_id::QProvingJobDataID, + proof_store::QProofStoreReaderSync, +}; +use city_rollup_common::qworker::{ + dump::QJobWithDependenciesSerialized, + job_id::{ProvingJobDataType, QJobTopic}, + job_witnesses::inspect::{QJobProofPublicInputs, QJobWitnessWithId}, +}; +use city_rollup_core_orchestrator::debug::scenario::{ + actors::job_planner::plan_jobs, block_planner::transition::CityOpJobIds, +}; +use plonky2::{ + field::goldilocks_field::GoldilocksField, hash::hash_types::RichField, + plonk::config::PoseidonGoldilocksConfig, +}; +use serde::{Deserialize, Serialize}; + +use crate::dump::{get_rpc_proof_dependencies, BlockProofStoreDump, DumpProofStoreConfig}; + +#[derive(Serialize, Deserialize, Clone)] +#[serde(bound = "")] +pub struct InspectDumpOutput { + pub dependency_map: Option, + pub job_config: Option, + pub signature_proof_dependency_ids: Option>, + pub proof_witnesses: Option>>, + pub proof_public_inputs: Option>>, +} +pub fn run_inspect_l2_dump(args: &InspectL2DumpArgs) -> anyhow::Result<()> { + type F = GoldilocksField; + type C = PoseidonGoldilocksConfig; + const D: usize = 2; + + let root = std::env::current_dir()?; + let input_path = root.join(args.input.clone()).display().to_string(); + let input_bytes = std::fs::read(input_path)?; + let dump: BlockProofStoreDump = bincode::deserialize(&input_bytes)?; + let dump_config = dump.config.clone(); + let block_op_job_ids = + CityOpJobIds::dummy_from_config(dump_config.checkpoint_id, &dump_config.job_config); + let num_input_witnesses = dump_config.job_config.add_deposit_count + 1; + + let mut proof_store = dump.store.clone(); + + let leaves = plan_jobs( + &mut proof_store, + &block_op_job_ids, + num_input_witnesses, + dump_config.checkpoint_id, + )?; + let dependency_map = dump_job_dependencies_from_store(&proof_store, &leaves)?; + + let dep_tree = dependency_map.get_dependency_tree_for_block(dump_config.checkpoint_id); + let rpc_signature_proof_dependencies_ids = get_rpc_proof_dependencies(&dump_config)?; + let all_job_ids = dep_tree.get_all_dependencies(); + let proof_job_input_witness_ids = all_job_ids + .iter() + .filter(|id| { + id.topic.eq(&QJobTopic::GenerateStandardProof) + && id.data_type.eq(&ProvingJobDataType::InputWitness) + }) + .collect::>(); + + let proof_job_output_ids = proof_job_input_witness_ids + .iter() + .map(|id| id.get_output_id()) + .collect::>(); + + let all_proof_result_ids: Vec = [ + rpc_signature_proof_dependencies_ids.clone(), + proof_job_output_ids, + ] + .concat(); + + let mut result = InspectDumpOutput:: { + dependency_map: None, + job_config: None, + signature_proof_dependency_ids: None, + proof_witnesses: None, + proof_public_inputs: None, + }; + if args.data.contains(&QDumpInspectionData::DependencyMap) { + result.dependency_map = Some(dep_tree.to_serialized()); + } + if args.data.contains(&QDumpInspectionData::JobConfig) { + result.job_config = Some(dump_config); + } + if args.data.contains(&QDumpInspectionData::SignatureProofIds) { + result.signature_proof_dependency_ids = Some(rpc_signature_proof_dependencies_ids); + } + if args.data.contains(&QDumpInspectionData::ProofWitnesses) { + let proof_witnesses = proof_job_input_witness_ids + .into_iter() + .map(|id| { + let data = proof_store.get_bytes_by_id(*id)?; + QJobWitnessWithId::::try_deserialize_witness(*id, &data) + }) + .collect::>>>()?; + result.proof_witnesses = Some(proof_witnesses); + } + + if args.data.contains(&QDumpInspectionData::ProofPublicInputs) { + /* + let proof_public_inputs = all_proof_result_ids.iter().map(|id| { + println!("Getting proof public inputs for id: {:?}", id); + let data = proof_store.get_proof_by_id::(*id)?; + Ok(QJobProofPublicInputs::new(*id, data.public_inputs.clone())) + }).collect::>>>()?; + result.proof_public_inputs = Some(proof_public_inputs); + */ + + let proof_public_inputs = all_proof_result_ids + .iter() + .map(|id| { + let data = proof_store.get_proof_by_id::(*id); + if data.is_err() { + None + } else { + Some(QJobProofPublicInputs::new(*id, data.unwrap().public_inputs)) + } + }) + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + .collect::>>(); + result.proof_public_inputs = Some(proof_public_inputs); + } + + if args.output.is_empty() { + println!("{}", serde_json::to_string_pretty(&result)?); + } else { + let output_path = root.join(args.output.clone()).display().to_string(); + std::fs::write(output_path, serde_json::to_string_pretty(&result)?)?; + } + Ok(()) +} diff --git a/city_rollup_core_worker_qbench/src/lib.rs b/city_rollup_core_worker_qbench/src/lib.rs new file mode 100644 index 00000000..3c477257 --- /dev/null +++ b/city_rollup_core_worker_qbench/src/lib.rs @@ -0,0 +1,3 @@ +pub mod dump; +pub mod qbench; +pub mod inspect; \ No newline at end of file diff --git a/city_rollup_core_worker_qbench/src/qbench.rs b/city_rollup_core_worker_qbench/src/qbench.rs new file mode 100644 index 00000000..e39f32fd --- /dev/null +++ b/city_rollup_core_worker_qbench/src/qbench.rs @@ -0,0 +1,85 @@ +use crate::dump::BlockProofStoreDump; +use city_common::cli::{args::QBenchArgs, modes::QWorkerMode}; +use city_rollup_circuit::worker::toolbox::root::CRWorkerToolboxRootCircuits; +use city_rollup_common::{ + actors::{simple::events::CityEventProcessorMemory, traits::WorkerEventTransmitterSync}, + config::sighash_wrapper_config::SIGHASH_WHITELIST_TREE_ROOT, + introspection::rollup::constants::get_network_magic_for_str, + qworker::job_id::QWorkerJobBenchmark, +}; +use city_rollup_core_orchestrator::debug::scenario::actors::job_planner::plan_jobs; +use city_rollup_core_orchestrator::debug::scenario::block_planner::transition::CityOpJobIds; +use city_rollup_core_worker::actors::simple::SimpleActorWorker; +use plonky2::plonk::config::PoseidonGoldilocksConfig; + +pub fn run_worker_qbench( + args: &QBenchArgs, + dumps: Vec, +) -> anyhow::Result> { + type C = PoseidonGoldilocksConfig; + const D: usize = 2; + println!("Initializing QBench (this may take a few minutes)..."); + let network_magic = get_network_magic_for_str(args.network.to_string())?; + + let mut toolbox = + CRWorkerToolboxRootCircuits::::new(network_magic, SIGHASH_WHITELIST_TREE_ROOT); + + gnark_plonky2_wrapper::initialize(&format!( + "{}/.city-rollup/keystore/", + home::home_dir().unwrap().display() + ))?; + println!("QBench Initialization Complete!"); + let mut benchmark_results: Vec = Vec::new(); + for dump in dumps.into_iter() { + let dump_config = dump.config.clone(); + let block_op_job_ids = + CityOpJobIds::dummy_from_config(dump_config.checkpoint_id, &dump_config.job_config); + let num_input_witnesses = dump_config.job_config.add_deposit_count + 1; + + let mut proof_store = dump.store.clone(); + let mut event_processor = CityEventProcessorMemory::new_with_config(true); + + println!("Running qbench with {} iterations", args.num_iterations); + + for _ in 0..args.num_iterations { + let leaves = plan_jobs( + &mut proof_store, + &block_op_job_ids, + num_input_witnesses, + dump_config.checkpoint_id, + )?; + event_processor.enqueue_jobs(&leaves)?; + while !event_processor.job_queue.is_empty() { + SimpleActorWorker::process_next_job( + &mut proof_store, + &mut event_processor, + &mut toolbox, + QWorkerMode::All, + )? + } + } + benchmark_results.append(&mut event_processor.benchmarks); + } + Ok(benchmark_results) +} +pub fn run_qbench(args: &QBenchArgs) -> anyhow::Result<()> { + let root = std::env::current_dir()?; + let input_paths = args + .input + .iter() + .map(|input| root.join(input.clone()).display().to_string()) + .collect::>(); + let output_path = root.join(args.output.clone()).display().to_string(); + let mut dumps: Vec = Vec::new(); + for input_path in input_paths.iter() { + let input_bytes = std::fs::read(input_path)?; + let dump: BlockProofStoreDump = bincode::deserialize(&input_bytes)?; + dumps.push(dump); + } + let results = run_worker_qbench(args, dumps)?; + + let results_json_bytes = serde_json::to_vec_pretty(&results)?; + std::fs::write(output_path, results_json_bytes)?; + + Ok(()) +} diff --git a/city_rollup_dev_cli/examples/dump_proof_store.rs b/city_rollup_dev_cli/examples/dump_proof_store.rs new file mode 100644 index 00000000..a8967f1c --- /dev/null +++ b/city_rollup_dev_cli/examples/dump_proof_store.rs @@ -0,0 +1,106 @@ +use std::path::PathBuf; + +use city_redis_store::RedisStore; +use city_rollup_common::qworker::{ + dump::dump_job_dependencies_from_store, + job_id::{QProvingJobDataID, QProvingJobDataIDSerializedWrapped}, + memory_proof_store::SimpleProofStoreMemory, + proof_store::{QProofStore, QProofStoreReaderSync}, +}; +use city_rollup_core_orchestrator::debug::scenario::{ + actors::job_planner::plan_jobs, + block_planner::transition::{CityOpJobConfig, CityOpJobIds}, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone)] +struct DumpProofStoreConfig { + pub checkpoint_id: u64, + pub rpc_node_id: u32, + pub job_config: CityOpJobConfig, +} +fn mirror_proof_store(keys: &[QProvingJobDataID], source: &PSource, destination: &mut PDestination) -> anyhow::Result<()> { + for key in keys { + let data = source.get_bytes_by_id(*key)?; + destination.set_bytes_by_id(*key, &data)?; + } + Ok(()) +} +// this only works when running a single rpc node with a the node id "rpc_node_id", which is ok because this is for debugging only +fn get_rpc_proof_dependencies( + config: &DumpProofStoreConfig, +) -> anyhow::Result> { + let token_transfer_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::transfer_signature_proof(config.rpc_node_id, config.checkpoint_id, i as u32) + }); + + let claim_deposit_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::claim_deposit_l1_signature_proof( + config.rpc_node_id, + config.checkpoint_id, + i as u32, + ) + }); + + let withdrawal_signature_proof_ids = (0..config.job_config.token_transfer_count).map(|i| { + QProvingJobDataID::withdrawal_signature_proof(config.rpc_node_id, config.checkpoint_id, i as u32) + }); + + Ok(token_transfer_signature_proof_ids + .chain(claim_deposit_signature_proof_ids) + .chain(withdrawal_signature_proof_ids) + .collect()) +} +fn dump_proof_store( + config: &DumpProofStoreConfig, + real_store: &PS, +) -> anyhow::Result { + let mut mirror_store = SimpleProofStoreMemory::new(); + let block_op_job_ids = + CityOpJobIds::dummy_from_config(config.checkpoint_id, &config.job_config); + let num_input_witnesses = config.job_config.add_deposit_count + 1; + let leaves = plan_jobs( + &mut mirror_store, + &block_op_job_ids, + num_input_witnesses, + config.checkpoint_id, + )?; + println!( + "leaf_jobs: {}", + serde_json::to_string( + &leaves + .iter() + .map(|x| QProvingJobDataIDSerializedWrapped(x.to_fixed_bytes())) + .collect::>() + )? + ); + + + let dependency_map = dump_job_dependencies_from_store(real_store, &leaves)?; + + let dep_tree = dependency_map.get_dependency_tree_for_block(config.checkpoint_id); + let rpc_signature_proof_dependencies = get_rpc_proof_dependencies(config)?; + println!("mirroring signature proofs"); + mirror_proof_store(&rpc_signature_proof_dependencies, real_store, &mut mirror_store)?; + + + let proof_witnesses = dep_tree.get_all_dependencies(); + println!("mirroring proof witnesses"); + mirror_proof_store(&proof_witnesses, real_store, &mut mirror_store)?; + + + Ok(mirror_store) +} +fn main() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = format!("{}/examples/dump_proof_store_block_config.json", root.display()); + let output_path = format!("{}/examples/dump_proof_init.bin", root.display()); + let file_data = std::fs::read(path).unwrap(); + let config: DumpProofStoreConfig = serde_json::from_slice(&file_data).unwrap(); + let redis_uri = "redis://localhost:6379/0"; + + let real_store = RedisStore::new(redis_uri).unwrap(); + let result = dump_proof_store(&config, &real_store).unwrap(); + + std::fs::write(output_path, &result.to_serialized_bytes().unwrap()).unwrap(); +} diff --git a/city_rollup_dev_cli/examples/dump_proof_store_block_config.json b/city_rollup_dev_cli/examples/dump_proof_store_block_config.json new file mode 100644 index 00000000..0c8832da --- /dev/null +++ b/city_rollup_dev_cli/examples/dump_proof_store_block_config.json @@ -0,0 +1,12 @@ +{ + "checkpoint_id": 4, + "rpc_node_id": 0, + "job_config": { + "register_user_count": 4, + "claim_deposit_count": 2, + "token_transfer_count": 4, + "add_withdrawal_count": 4, + "process_withdrawal_count": 4, + "add_deposit_count": 2 + } +} \ No newline at end of file diff --git a/city_rollup_dev_cli/examples/dump_proof_store_leaf_jobs.json b/city_rollup_dev_cli/examples/dump_proof_store_leaf_jobs.json new file mode 100644 index 00000000..7483d3fa --- /dev/null +++ b/city_rollup_dev_cli/examples/dump_proof_store_leaf_jobs.json @@ -0,0 +1 @@ +["0002000000000000002121cf000000000000000000000000","0002000000000000000000cf000000000000000000000000","0002000000000000000000cf000000000000010000000000","00020000000000000032dd00000000000000000000000000","00020000000000000033dd00000000000000000000000000","00020000000000000034dd00000000000000000000000000","00020000000000000035dd00000000000000000000000000","00020000000000000031dd00000000000000000000000000"] \ No newline at end of file diff --git a/city_rollup_dev_cli/examples/fblockredis.rs b/city_rollup_dev_cli/examples/fblockredis.rs index 69c0feaf..543ea811 100644 --- a/city_rollup_dev_cli/examples/fblockredis.rs +++ b/city_rollup_dev_cli/examples/fblockredis.rs @@ -5,6 +5,7 @@ use city_common::{ }; use city_crypto::hash::{base_types::hash256::Hash256, qhashout::QHashOut}; use city_redis_store::RedisStore; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_common::{ actors::{ rpc_processor::QRPCProcessor, @@ -19,7 +20,7 @@ use city_rollup_common::{ qworker::fingerprints::CRWorkerToolboxCoreCircuitFingerprints, }; use city_rollup_core_orchestrator::{ - debug::scenario::{actors::simple::SimpleActorOrchestrator, wallet::DebugScenarioWallet}, + debug::scenario::actors::simple::SimpleActorOrchestrator, event_receiver::CityEventReceiver, }; use city_rollup_core_worker::event_processor::CityEventProcessor; @@ -64,7 +65,7 @@ fn run_full_block() -> anyhow::Result<()> { timer.lap("start creating wallets"); - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); let genesis_funder_public_key = wallet.add_secp256k1_private_key(Hash256( hex_literal::hex!("133700f4676a0d0e16aaced646ed693626fcf1329db55be8eee13ad8df001337"), diff --git a/city_rollup_dev_cli/examples/full_block.rs b/city_rollup_dev_cli/examples/full_block.rs index 5e981108..d091bc02 100644 --- a/city_rollup_dev_cli/examples/full_block.rs +++ b/city_rollup_dev_cli/examples/full_block.rs @@ -5,9 +5,9 @@ use city_crypto::hash::{ base_types::{felt252::felt252_hashout_to_hash256_le, hash256::Hash256}, qhashout::QHashOut, }; -use city_rollup_circuit::worker::{ +use city_rollup_circuit::{wallet::memory::CityMemoryWallet, worker::{ prover::QWorkerStandardProver, toolbox::root::CRWorkerToolboxRootCircuits, -}; +}}; use city_rollup_common::{ actors::{requested_actions::CityScenarioRequestedActions, rpc_processor::QRPCProcessor}, api::data::{block::rpc_request::CityRegisterUserRPCRequest, store::CityL2BlockState}, @@ -19,7 +19,6 @@ use city_rollup_common::{ }; use city_rollup_core_orchestrator::debug::scenario::{ block_planner::planner::CityOrchestratorBlockPlanner, sighash::finalizer::SigHashFinalizer, - wallet::DebugScenarioWallet, }; use city_store::store::{city::base::CityStore, sighash::SigHashMerkleTree}; use kvq::memory::simple::KVQSimpleMemoryBackingStore; @@ -67,7 +66,7 @@ fn prove_block_demo(hints: &[BlockSpendIntrospectionHint]) -> anyhow::Result<()> timer.lap("start creating wallets"); - let mut wallet = DebugScenarioWallet::::new(); + let mut wallet = CityMemoryWallet::::new(); let _deposit_0_public_key = wallet.add_secp256k1_private_key(Hash256(hex_literal::hex!( "07e5cae38a63f487667075c54cb7791b86179e3becd9198e5ee0557eeffcda31" diff --git a/city_rollup_dev_cli/examples/full_block_new.rs b/city_rollup_dev_cli/examples/full_block_new.rs deleted file mode 100644 index 44a18262..00000000 --- a/city_rollup_dev_cli/examples/full_block_new.rs +++ /dev/null @@ -1,286 +0,0 @@ -use std::{thread::sleep, time::Duration}; - -use city_common::{ - cli::{message::CITY_ROLLUP_BANNER, modes::QWorkerMode}, logging::debug_timer::DebugTimer, units::UNIT_BTC, -}; -use city_crypto::hash::{base_types::hash256::Hash256, qhashout::QHashOut}; -use city_rollup_circuit::worker::toolbox::root::CRWorkerToolboxRootCircuits; -use city_rollup_common::{ - actors::{simple::events::CityEventProcessorMemory, traits::OrchestratorRPCEventSenderSync}, - api::data::{block::rpc_request::CityRegisterUserRPCRequest, store::CityL2BlockState}, - introspection::rollup::constants::NETWORK_MAGIC_DOGE_REGTEST, - link::{ - data::BTCAddress160, link_api::BTCLinkAPI, traits::QBitcoinAPIFunderSync, - tx::setup_genesis_block, - }, - qworker::memory_proof_store::SimpleProofStoreMemory, -}; -use city_rollup_core_orchestrator::debug::{ - coordinator::core::DevMemoryCoordinatatorRPCQueue, - scenario::{actors::simple::SimpleActorOrchestrator, wallet::DebugScenarioWallet}, -}; -use city_rollup_core_worker::actors::simple::SimpleActorWorker; -use city_store::store::{city::base::CityStore, sighash::SigHashMerkleTree}; -use kvq::memory::simple::KVQSimpleMemoryBackingStore; -use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; -fn run_full_block() -> anyhow::Result<()> { - tracing::info!("{}", CITY_ROLLUP_BANNER); - let mut api = BTCLinkAPI::new_str( - "http://devnet:devnet@localhost:1337/bitcoin-rpc/?network=dogeRegtest", - "http://localhost:1337/api", - ); - - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - type F = GoldilocksField; - type S = KVQSimpleMemoryBackingStore; - type PS = SimpleProofStoreMemory; - type CityWorker = SimpleActorWorker; - type CityOrchestrator = SimpleActorOrchestrator; - - let network_magic = NETWORK_MAGIC_DOGE_REGTEST; - - let sighash_whitelist_tree = SigHashMerkleTree::new(); - - let mut proof_store = PS::new(); - let mut store = S::new(); - let mut timer = DebugTimer::new("prove_block_demo"); - let mut worker_event_processor = CityEventProcessorMemory::new(); - let mut rpc_queue = DevMemoryCoordinatatorRPCQueue::::new(); - - /* - let start_state_root = CityStore::get_city_root(&store, 1)?; - tracing::info!( - "start_state_root: {} ({:?})", - start_state_root.to_string(), - start_state_root.0 - );*/ - - timer.lap("start creating wallets"); - - let mut wallet = DebugScenarioWallet::::new_fast_setup(); - - let genesis_funder_public_key = wallet.add_secp256k1_private_key(Hash256( - hex_literal::hex!("133700f4676a0d0e16aaced646ed693626fcf1329db55be8eee13ad8df001337"), - ))?; - let genesis_funder_address = BTCAddress160::from_p2pkh_key(genesis_funder_public_key); - - let deposit_0_public_key = wallet.add_secp256k1_private_key(Hash256(hex_literal::hex!( - "e6baf19a8b0b9b8537b9354e178a0a42d0887371341d4b2303537c5d18d7bb87" - )))?; - let deposit_0_address = BTCAddress160::from_p2pkh_key(deposit_0_public_key); - - let deposit_1_public_key = wallet.add_secp256k1_private_key(Hash256(hex_literal::hex!( - "51dfec6b389f5f033bbe815d5df995a20851227fd845a3be389ca9ad2b6924f0" - )))?; - let deposit_1_address = BTCAddress160::from_p2pkh_key(deposit_1_public_key); - - timer.lap("end creating wallets"); - - timer.lap("start setup initial state"); - let block_0_state = CityL2BlockState { - checkpoint_id: 0, - next_add_withdrawal_id: 0, - next_process_withdrawal_id: 0, - next_deposit_id: 0, - total_deposits_claimed_epoch: 0, - next_user_id: 0, - end_balance: 0, - }; - let block_1_state = CityL2BlockState { - checkpoint_id: 1, - next_add_withdrawal_id: 0, - next_process_withdrawal_id: 0, - next_deposit_id: 0, - total_deposits_claimed_epoch: 0, - next_user_id: 0, - end_balance: 0, - }; - - CityStore::set_block_state(&mut store, &block_0_state)?; - CityStore::set_block_state(&mut store, &block_1_state)?; - let genesis_state_hash = CityStore::get_city_root(&store, 0)?; - let setup_fee = 100000 * 500; - let fund_genesis_txid = api.fund_address_from_random_p2pkh_address( - genesis_funder_address, - 101 * UNIT_BTC + setup_fee * 4, - )?; - - api.mine_blocks(1)?; - let txid_fund_genesis = setup_genesis_block( - &api, - &wallet.secp256k1_wallet, - genesis_funder_address.address, - fund_genesis_txid, - setup_fee, - genesis_state_hash.to_felt252_hash256(), - )?; - tracing::info!( - "funded genesis block with txid: {}", - txid_fund_genesis.to_hex_string() - ); - //tracing::info!("txid_fund_genesis: {}", txid_fund_genesis.to_hex_string()); - let block_2_address = - BTCAddress160::new_p2sh(CityStore::get_city_block_deposit_address(&store, 2)?); - - api.mine_blocks(1)?; - tracing::info!("block_2_address: {}", block_2_address.to_string()); - - timer.lap("start creating wallets"); - let user_0_public_key = wallet.add_zk_private_key(QHashOut::from_values(100, 100, 100, 100)); - let user_1_public_key = wallet.add_zk_private_key(QHashOut::from_values(101, 101, 101, 101)); - let user_2_public_key = wallet.add_zk_private_key(QHashOut::from_values(102, 102, 102, 102)); - wallet.setup_circuits(); - timer.lap("end creating wallets"); - - timer.lap("start creating worker"); - let root_toolbox = - CRWorkerToolboxRootCircuits::::new(network_magic, sighash_whitelist_tree.root); - let fingerprints = root_toolbox.core.fingerprints.clone(); - timer.lap("end creating worker"); - - let mut checkpoint_id = 2; - timer.lap("start fund block 2"); - api.fund_address_from_known_p2pkh_address( - &wallet.secp256k1_wallet, - deposit_0_address, - block_2_address, - 10 * UNIT_BTC, - )?; - api.fund_address_from_known_p2pkh_address( - &wallet.secp256k1_wallet, - deposit_1_address, - block_2_address, - 15 * UNIT_BTC, - )?; - api.mine_blocks(10)?; - timer.lap("waiting for deposits for 10 seconds..."); - sleep(Duration::from_millis(1000 * 10)); - timer.lap("end fund block 2"); - - timer.lap("start prepare block 2 events"); - let register_user_rpc_events = - CityRegisterUserRPCRequest::new_batch(&[user_0_public_key, user_1_public_key]); - let _ = register_user_rpc_events - .into_iter() - .map(|x| rpc_queue.notify_rpc_register_user(&x)) - .collect::>>()?; - timer.lap("end prepare block 2 events"); - let mut requested_actions = - rpc_queue.get_requested_actions_from_rpc(&mut proof_store, checkpoint_id)?; - - let orchestrator_result_step_1 = CityOrchestrator::step_1_produce_block_enqueue_jobs( - &mut proof_store, - &mut store, - &mut requested_actions, - &mut worker_event_processor, - &mut api, - &fingerprints, - &sighash_whitelist_tree, - )?; - let end_state_root = CityStore::get_city_root(&store, 2)?; - tracing::info!( - "end_state_root: {} ({:?})", - end_state_root.to_string(), - end_state_root.0 - ); - loop { - if worker_event_processor.job_queue.is_empty() { - break; - } - CityWorker::process_next_job(&mut proof_store, &mut worker_event_processor, &root_toolbox, QWorkerMode::All)?; - } - api.mine_blocks(1)?; - let orchestrator_result_step_2 = CityOrchestrator::step_2_produce_block_finalize_and_transact( - &mut proof_store, - &mut api, - &orchestrator_result_step_1, - )?; - tracing::info!( - "produced block, sent to : {}", - orchestrator_result_step_2.to_hex_string() - ); - api.mine_blocks(1)?; - checkpoint_id = 3; - tracing::info!("starting block {}", checkpoint_id); - - timer.lap("start prepare block 3 events"); - let register_user_rpc_events = CityRegisterUserRPCRequest::new_batch(&[user_2_public_key]); - let _ = register_user_rpc_events - .into_iter() - .map(|x| rpc_queue.notify_rpc_register_user(&x)) - .collect::>>()?; - - rpc_queue.notify_rpc_claim_deposit(&wallet.sign_claim_deposit( - network_magic, - 0, - &CityStore::::get_deposit_by_id(&store, checkpoint_id, 0)?, - )?)?; - rpc_queue.notify_rpc_claim_deposit(&wallet.sign_claim_deposit( - network_magic, - 1, - &CityStore::::get_deposit_by_id(&store, checkpoint_id, 1)?, - )?)?; - - rpc_queue.notify_rpc_token_transfer(&wallet.sign_l2_transfer( - user_0_public_key, - network_magic, - 0, - 1, - 2 * UNIT_BTC, - 1, - )?)?; - rpc_queue.notify_rpc_token_transfer(&wallet.sign_l2_transfer( - user_1_public_key, - network_magic, - 1, - 2, - 5 * UNIT_BTC, - 1, - )?)?; - - timer.lap("end prepare block 3 events"); - - let mut requested_actions = - rpc_queue.get_requested_actions_from_rpc(&mut proof_store, checkpoint_id)?; - - let orchestrator_result_step_1 = CityOrchestrator::step_1_produce_block_enqueue_jobs( - &mut proof_store, - &mut store, - &mut requested_actions, - &mut worker_event_processor, - &mut api, - &fingerprints, - &sighash_whitelist_tree, - )?; - /*let end_state_root = CityStore::get_city_root(&store, 2)?; - tracing::info!( - "end_state_root: {} ({:?})", - end_state_root.to_string(), - end_state_root.0 - );*/ - loop { - if worker_event_processor.job_queue.is_empty() { - break; - } - CityWorker::process_next_job(&mut proof_store, &mut worker_event_processor, &root_toolbox, QWorkerMode::All)?; - } - api.mine_blocks(1)?; - let orchestrator_result_step_2 = CityOrchestrator::step_2_produce_block_finalize_and_transact( - &mut proof_store, - &mut api, - &orchestrator_result_step_1, - )?; - tracing::info!( - "produced block, sent to : {}", - orchestrator_result_step_2.to_hex_string() - ); - api.mine_blocks(1)?; - checkpoint_id = 4; - tracing::info!("starting block {}", checkpoint_id); - Ok(()) -} - -fn main() { - run_full_block().unwrap(); -} diff --git a/city_rollup_dev_cli/examples/full_block_redis.rs b/city_rollup_dev_cli/examples/full_block_redis.rs index 16b2cae3..af509d3f 100644 --- a/city_rollup_dev_cli/examples/full_block_redis.rs +++ b/city_rollup_dev_cli/examples/full_block_redis.rs @@ -5,7 +5,7 @@ use city_common::{ }; use city_crypto::hash::{base_types::hash256::Hash256, qhashout::QHashOut}; use city_redis_store::RedisStore; -use city_rollup_circuit::worker::toolbox::root::CRWorkerToolboxRootCircuits; +use city_rollup_circuit::{wallet::memory::CityMemoryWallet, worker::toolbox::root::CRWorkerToolboxRootCircuits}; use city_rollup_common::{ actors::{ rpc_processor::QRPCProcessor, @@ -19,15 +19,11 @@ use city_rollup_common::{ }, }; use city_rollup_core_orchestrator::{ - debug::scenario::{actors::simple::SimpleActorOrchestrator, wallet::DebugScenarioWallet}, + debug::scenario::actors::simple::SimpleActorOrchestrator, event_receiver::CityEventReceiver, }; -use city_rollup_core_worker::{ - event_processor::CityEventProcessor, -}; -use city_rollup_worker_dispatch::{ - implementations::redis::RedisQueue, -}; +use city_rollup_core_worker::event_processor::CityEventProcessor; +use city_rollup_worker_dispatch::implementations::redis::RedisQueue; use city_store::store::{city::base::CityStore, sighash::SigHashMerkleTree}; use kvq::memory::simple::KVQSimpleMemoryBackingStore; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; @@ -66,7 +62,7 @@ fn run_full_block() -> anyhow::Result<()> { timer.lap("start creating wallets"); - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); let genesis_funder_public_key = wallet.add_secp256k1_private_key(Hash256( hex_literal::hex!("133700f4676a0d0e16aaced646ed693626fcf1329db55be8eee13ad8df001337"), diff --git a/city_rollup_dev_cli/examples/full_block_v2.rs b/city_rollup_dev_cli/examples/full_block_v2.rs index a66f6786..c640c9c7 100644 --- a/city_rollup_dev_cli/examples/full_block_v2.rs +++ b/city_rollup_dev_cli/examples/full_block_v2.rs @@ -5,9 +5,9 @@ use city_crypto::hash::{ base_types::{felt252::felt252_hashout_to_hash256_le, hash256::Hash256}, qhashout::QHashOut, }; -use city_rollup_circuit::worker::{ +use city_rollup_circuit::{wallet::memory::CityMemoryWallet, worker::{ prover::QWorkerStandardProver, toolbox::root::CRWorkerToolboxRootCircuits, -}; +}}; use city_rollup_common::{ actors::{requested_actions::CityScenarioRequestedActions, rpc_processor::QRPCProcessor}, api::data::{ @@ -25,7 +25,6 @@ use city_rollup_common::{ }; use city_rollup_core_orchestrator::debug::scenario::{ block_planner::planner::CityOrchestratorBlockPlanner, sighash::finalizer::SigHashFinalizer, - wallet::DebugScenarioWallet, }; use city_store::store::{city::base::CityStore, sighash::SigHashMerkleTree}; use kvq::memory::simple::KVQSimpleMemoryBackingStore; @@ -51,7 +50,7 @@ fn prove_block_demo(hints: &[BlockSpendIntrospectionHint]) -> anyhow::Result<()> timer.lap("start creating wallets"); - let mut wallet = DebugScenarioWallet::::new(); + let mut wallet = CityMemoryWallet::::new(); let _deposit_0_public_key = wallet.add_secp256k1_private_key(Hash256(hex_literal::hex!( "e6baf19a8b0b9b8537b9354e178a0a42d0887371341d4b2303537c5d18d7bb87" diff --git a/city_rollup_dev_cli/examples/sign_secp256k1.rs b/city_rollup_dev_cli/examples/sign_secp256k1.rs index cf42eba6..cad1a2f6 100644 --- a/city_rollup_dev_cli/examples/sign_secp256k1.rs +++ b/city_rollup_dev_cli/examples/sign_secp256k1.rs @@ -1,7 +1,7 @@ use city_common::logging::debug_timer::DebugTimer; use city_crypto::hash::{base_types::hash256::Hash256, qhashout::QHashOut}; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_common::introspection::rollup::constants::NETWORK_MAGIC_DOGE_REGTEST; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; use plonky2::{ field::{goldilocks_field::GoldilocksField, types::Field}, hash::hash_types::HashOut, @@ -20,7 +20,7 @@ fn prove_sig_demo() -> anyhow::Result<()> { timer.lap("start creating wallets"); - let mut wallet = DebugScenarioWallet::::new(); + let mut wallet = CityMemoryWallet::::new(); let deposit_0_public_key = wallet.add_secp256k1_private_key(Hash256(hex_literal::hex!( "e6baf19a8b0b9b8537b9354e178a0a42d0887371341d4b2303537c5d18d7bb87" diff --git a/city_rollup_rpc_provider/Cargo.toml b/city_rollup_rpc_provider/Cargo.toml index f83e59e8..e4050387 100644 --- a/city_rollup_rpc_provider/Cargo.toml +++ b/city_rollup_rpc_provider/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" reqwest = { workspace = true } async-trait = { workspace = true } city_rollup_core_api = { path = "../city_rollup_core_api" } +city_common = { path = "../city_common" } city_crypto = { path = "../city_crypto" } city_rollup_common = { path = "../city_rollup_common" } city_rollup_core_node = { path = "../city_rollup_core_node" } diff --git a/city_rollup_rpc_provider/src/lib.rs b/city_rollup_rpc_provider/src/lib.rs index 0808d82b..671a1d0f 100644 --- a/city_rollup_rpc_provider/src/lib.rs +++ b/city_rollup_rpc_provider/src/lib.rs @@ -1,12 +1,13 @@ use std::sync::Arc; +use city_common::data::{kv::SimpleKVPair, u8bytes::U8Bytes}; use city_crypto::hash::base_types::hash160::Hash160; use city_crypto::hash::base_types::hash256::Hash256; use city_macros::{city_external_rpc_call, city_external_rpc_call_sync, city_rpc_call, city_rpc_call_sync}; -use city_rollup_common::api::data::{ +use city_rollup_common::{api::data::{ block::rpc_request::*, - store::{CityL1Deposit, CityL1Withdrawal, CityL2BlockState, CityUserState}, -}; + store::{CityL1DepositJSON, CityL1Withdrawal, CityL2BlockState, CityUserState}, +}, qworker::job_id::QProvingJobDataIDSerializedWrapped}; use city_rollup_core_node::rpc::{ ExternalRequestParams, Id, RequestParams, ResponseResult, RpcParams, RpcRequest, RpcResponse, Version, @@ -80,20 +81,20 @@ pub trait CityRpcProvider { &self, checkpoint_id: u64, deposit_id: u64, - ) -> anyhow::Result; + ) -> anyhow::Result; async fn get_deposits_by_id( &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> anyhow::Result>; + ) -> anyhow::Result>; - async fn get_deposit_by_txid(&self, transaction_id: Hash256) -> anyhow::Result; + async fn get_deposit_by_txid(&self, transaction_id: Hash256) -> anyhow::Result; async fn get_deposits_by_txid( &self, transaction_ids: Vec, - ) -> anyhow::Result>; + ) -> anyhow::Result>; async fn get_deposit_hash( &self, @@ -148,6 +149,16 @@ pub trait CityRpcProvider { withdrawal_id: u64, ) -> anyhow::Result; + async fn get_proof_store_value( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> anyhow::Result; + + async fn get_proof_store_values( + &self, + keys: &[QProvingJobDataIDSerializedWrapped], + ) -> anyhow::Result>>; + async fn register_user( &self, req: CityRegisterUserRPCRequest, @@ -203,20 +214,20 @@ pub trait CityRpcProviderSync { &self, checkpoint_id: u64, deposit_id: u64, - ) -> anyhow::Result; + ) -> anyhow::Result; fn get_deposits_by_id_sync( &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> anyhow::Result>; + ) -> anyhow::Result>; - fn get_deposit_by_txid_sync(&self, transaction_id: Hash256) -> anyhow::Result; + fn get_deposit_by_txid_sync(&self, transaction_id: Hash256) -> anyhow::Result; fn get_deposits_by_txid_sync( &self, transaction_ids: Vec, - ) -> anyhow::Result>; + ) -> anyhow::Result>; fn get_deposit_hash_sync( &self, @@ -271,6 +282,16 @@ pub trait CityRpcProviderSync { withdrawal_id: u64, ) -> anyhow::Result; + fn get_proof_store_value_sync( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> anyhow::Result; + + fn get_proof_store_values_sync( + &self, + keys: &[QProvingJobDataIDSerializedWrapped], + ) -> anyhow::Result>>; + fn register_user_sync( &self, req: CityRegisterUserRPCRequest, @@ -367,12 +388,12 @@ impl CityRpcProvider for RpcProvider { &self, checkpoint_id: u64, deposit_id: u64, - ) -> anyhow::Result { + ) -> anyhow::Result { city_external_rpc_call!( self, "cr_getDepositById", json!([checkpoint_id, deposit_id]), - CityL1Deposit + CityL1DepositJSON ) } @@ -380,33 +401,33 @@ impl CityRpcProvider for RpcProvider { &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> anyhow::Result> { + ) -> anyhow::Result> { city_external_rpc_call!( self, "cr_getDepositsById", json!([checkpoint_id, deposit_ids]), - Vec + Vec ) } - async fn get_deposit_by_txid(&self, transaction_id: Hash256) -> anyhow::Result { + async fn get_deposit_by_txid(&self, transaction_id: Hash256) -> anyhow::Result { city_external_rpc_call!( self, "cr_getDepositByTxid", json!([transaction_id]), - CityL1Deposit + CityL1DepositJSON ) } async fn get_deposits_by_txid( &self, transaction_ids: Vec, - ) -> anyhow::Result> { + ) -> anyhow::Result> { city_external_rpc_call!( self, "cr_getDepositsByTxid", json!([transaction_ids]), - Vec + Vec ) } @@ -544,6 +565,29 @@ impl CityRpcProvider for RpcProvider { ) } + async fn get_proof_store_value( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> anyhow::Result { + city_external_rpc_call!( + self, + "cr_getProofStoreValue", + json!([key]), + U8Bytes + ) + } + async fn get_proof_store_values( + &self, + keys: &[QProvingJobDataIDSerializedWrapped], + ) -> anyhow::Result>> { + city_external_rpc_call!( + self, + "cr_getProofStoreValues", + json!([keys]), + Vec> + ) + } + async fn register_user( &self, req: CityRegisterUserRPCRequest, @@ -583,7 +627,7 @@ impl CityRpcProviderSync for RpcProviderSync { fn get_user_ids_for_public_key_sync(&self, public_key: CityHash) -> anyhow::Result> { city_external_rpc_call_sync!(self, "cr_getUserIdsForPublicKey", json!([public_key]), Vec) } - + fn get_user_by_id_sync( &self, checkpoint_id: u64, @@ -649,12 +693,12 @@ impl CityRpcProviderSync for RpcProviderSync { &self, checkpoint_id: u64, deposit_id: u64, - ) -> anyhow::Result { + ) -> anyhow::Result { city_external_rpc_call_sync!( self, "cr_getDepositById", json!([checkpoint_id, deposit_id]), - CityL1Deposit + CityL1DepositJSON ) } @@ -662,33 +706,33 @@ impl CityRpcProviderSync for RpcProviderSync { &self, checkpoint_id: u64, deposit_ids: Vec, - ) -> anyhow::Result> { + ) -> anyhow::Result> { city_external_rpc_call_sync!( self, "cr_getDepositsById", json!([checkpoint_id, deposit_ids]), - Vec + Vec ) } - fn get_deposit_by_txid_sync(&self, transaction_id: Hash256) -> anyhow::Result { + fn get_deposit_by_txid_sync(&self, transaction_id: Hash256) -> anyhow::Result { city_external_rpc_call_sync!( self, "cr_getDepositByTxid", json!([transaction_id]), - CityL1Deposit + CityL1DepositJSON ) } fn get_deposits_by_txid_sync( &self, transaction_ids: Vec, - ) -> anyhow::Result> { + ) -> anyhow::Result> { city_external_rpc_call_sync!( self, "cr_getDepositsByTxid", json!([transaction_ids]), - Vec + Vec ) } @@ -826,6 +870,30 @@ impl CityRpcProviderSync for RpcProviderSync { ) } + fn get_proof_store_value_sync( + &self, + key: QProvingJobDataIDSerializedWrapped, + ) -> anyhow::Result { + city_external_rpc_call_sync!( + self, + "cr_getProofStoreValue", + json!([key]), + U8Bytes + ) + } + + fn get_proof_store_values_sync( + &self, + keys: &[QProvingJobDataIDSerializedWrapped], + ) -> anyhow::Result>> { + city_external_rpc_call_sync!( + self, + "cr_getProofStoreValues", + json!([keys]), + Vec> + ) + } + fn register_user_sync( &self, req: CityRegisterUserRPCRequest, @@ -853,4 +921,4 @@ impl CityRpcProviderSync for RpcProviderSync { ) -> anyhow::Result<()> { city_rpc_call_sync!(self, RequestParams::::TokenTransfer(req)) } -} \ No newline at end of file +} diff --git a/city_rollup_user_cli/Cargo.toml b/city_rollup_user_cli/Cargo.toml index 2f6d74c7..ec79e2db 100644 --- a/city_rollup_user_cli/Cargo.toml +++ b/city_rollup_user_cli/Cargo.toml @@ -18,6 +18,7 @@ city_crypto = { path = "../city_crypto" } city_common_circuit = { path = "../city_common_circuit" } city_rollup_circuit = { path = "../city_rollup_circuit" } city_common = { path = "../city_common" } +city_rollup_user_prover_api = { path = "../city_rollup_user_prover_api" } city_rollup_common = { path = "../city_rollup_common" } city_rollup_core_node = { path = "../city_rollup_core_node" } city_rollup_core_orchestrator = { path = "../city_rollup_core_orchestrator" } diff --git a/city_rollup_user_cli/examples/ex1.rs b/city_rollup_user_cli/examples/ex1.rs new file mode 100644 index 00000000..0497c79d --- /dev/null +++ b/city_rollup_user_cli/examples/ex1.rs @@ -0,0 +1,61 @@ +use std::{collections::HashMap, sync::{mpsc, Arc, Mutex}, thread, time::Duration}; + +use city_crypto::hash::base_types::hash256::Hash256; +fn reverse_string(s: String) -> String { + s.chars().rev().collect() +} +#[derive(Clone, Debug)] +struct UserProverWorkerStore { + pub results: HashMap, +} +impl UserProverWorkerStore { + pub fn new() -> Self { + Self { + results: HashMap::new(), + } + } + pub fn get_result(&self, key: &Hash256) -> Option<&String> { + self.results.get(key) + } + pub fn set_result(&mut self, key: Hash256, value: String) { + self.results.insert(key, value); + } +} + +fn main() { + let (tx_main, rx_main) = mpsc::channel::(); + let (tx_worker, rx_worker) = mpsc::channel::(); + + let store = Arc::new(Mutex::new(UserProverWorkerStore::new())); + + let tx_main_in_worker = tx_main.clone(); + let store_in_worker = store.clone(); + thread::spawn(move || { + for recv in rx_worker { + let new_msg = reverse_string(recv); + let key = Hash256::rand(); + store_in_worker.lock().unwrap().set_result(key, new_msg); + tx_main_in_worker.send(key).unwrap(); + } + }); + + let tx_worker_in_api = tx_worker.clone(); + thread::spawn(move || { + let vals = vec![ + String::from("more"), + String::from("messages"), + String::from("for"), + String::from("you"), + ]; + + for val in vals { + tx_worker_in_api.send(val).unwrap(); + thread::sleep(Duration::from_secs(1)); + } + }); + + for received in rx_main { + let result = store.lock().unwrap().get_result(&received).unwrap().to_string(); + println!("Got: {result}"); + } +} diff --git a/city_rollup_user_cli/src/main.rs b/city_rollup_user_cli/src/main.rs index 5396e3eb..0938195f 100644 --- a/city_rollup_user_cli/src/main.rs +++ b/city_rollup_user_cli/src/main.rs @@ -18,6 +18,8 @@ use crate::subcommand::get_public_key; use crate::subcommand::random_wallet; use crate::subcommand::sign_hash; use crate::subcommand::repl; +use crate::subcommand::prover_rpc; + use crate::subcommand::Cli; use crate::subcommand::Commands; @@ -38,6 +40,7 @@ async fn main() -> Result<()> { Commands::GetPublicKey(args) => get_public_key::run(args).await?, Commands::RandomWallet(args) => random_wallet::run(args).await?, Commands::Repl(args) => repl::run(args).await?, + Commands::ProverRPC(args) => prover_rpc::run(args).await?, } diff --git a/city_rollup_user_cli/src/subcommand.rs b/city_rollup_user_cli/src/subcommand.rs index a450e10a..8c55e002 100644 --- a/city_rollup_user_cli/src/subcommand.rs +++ b/city_rollup_user_cli/src/subcommand.rs @@ -12,6 +12,7 @@ pub mod sign_hash; pub mod get_public_key; pub mod random_wallet; pub mod repl; +pub mod prover_rpc; #[derive(Parser)] pub struct Cli { @@ -31,4 +32,5 @@ pub enum Commands { GetPublicKey(city_common::cli::user_args::GetPublicKeyArgs), RandomWallet(city_common::cli::user_args::RandomWalletArgs), Repl(city_common::cli::user_args::RPCReplArgs), + ProverRPC(city_common::cli::user_args::ProverRPCArgs), } diff --git a/city_rollup_user_cli/src/subcommand/add_withdrawal.rs b/city_rollup_user_cli/src/subcommand/add_withdrawal.rs index 9e7a9f31..bf383d98 100644 --- a/city_rollup_user_cli/src/subcommand/add_withdrawal.rs +++ b/city_rollup_user_cli/src/subcommand/add_withdrawal.rs @@ -1,11 +1,11 @@ use anyhow::Result; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_rpc_provider::{CityRpcProvider, RpcProvider}; use std::str::FromStr; use city_common::cli::user_args::AddWithdrawalArgs; use city_crypto::hash::{base_types::hash160::Hash160, qhashout::QHashOut}; use city_rollup_common::{introspection::rollup::constants::get_network_magic_for_str, link::data::BTCAddress160}; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; const D: usize = 2; @@ -24,7 +24,7 @@ pub async fn run(args: AddWithdrawalArgs) -> Result<()> { let private_key = QHashOut::::from_str(&args.private_key) .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); let public_key = wallet.add_zk_private_key(private_key); diff --git a/city_rollup_user_cli/src/subcommand/claim_deposit.rs b/city_rollup_user_cli/src/subcommand/claim_deposit.rs index 45db170d..3a1bf38f 100644 --- a/city_rollup_user_cli/src/subcommand/claim_deposit.rs +++ b/city_rollup_user_cli/src/subcommand/claim_deposit.rs @@ -1,7 +1,7 @@ use city_common::cli::user_args::ClaimDepositArgs; use city_crypto::hash::base_types::hash256::Hash256; use city_rollup_common::introspection::rollup::constants::get_network_magic_for_str; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_rpc_provider::{CityRpcProvider, RpcProvider}; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; @@ -17,7 +17,7 @@ pub async fn run(args: ClaimDepositArgs) -> Result<()> { let network_magic = get_network_magic_for_str(args.network)?; - let mut wallet = DebugScenarioWallet::::new(); + let mut wallet = CityMemoryWallet::::new(); wallet.add_secp256k1_private_key(Hash256::from_hex_string(&args.private_key)?)?; @@ -26,7 +26,7 @@ pub async fn run(args: ClaimDepositArgs) -> Result<()> { let deposit = provider.get_deposit_by_txid(txid).await?; let city_claim_deposit_request = - wallet.sign_claim_deposit(network_magic, args.user_id, &deposit)?; + wallet.sign_claim_deposit(network_magic, args.user_id, &deposit.to_city_l1_deposit())?; provider .claim_deposit::(city_claim_deposit_request) diff --git a/city_rollup_user_cli/src/subcommand/prover_rpc.rs b/city_rollup_user_cli/src/subcommand/prover_rpc.rs new file mode 100644 index 00000000..e89d0e1c --- /dev/null +++ b/city_rollup_user_cli/src/subcommand/prover_rpc.rs @@ -0,0 +1,12 @@ +use city_common::cli::user_args::ProverRPCArgs; +use city_crypto::hash::base_types::hash256::Hash256; + +pub async fn run(args: ProverRPCArgs) -> anyhow::Result<()> { + let api_key = if args.api_key.is_empty() { + Hash256::rand() + } else { + Hash256::from_hex_string(&args.api_key).map_err(|_| anyhow::anyhow!("invalid api key (must be 32 bytes, hex encoded)"))? + }; + city_rollup_user_prover_api::run::run_server(args.prover_rpc_address, api_key).await?; + Ok(()) +} diff --git a/city_rollup_user_cli/src/subcommand/random_wallet.rs b/city_rollup_user_cli/src/subcommand/random_wallet.rs index ba2d3ce8..9e9a3c45 100644 --- a/city_rollup_user_cli/src/subcommand/random_wallet.rs +++ b/city_rollup_user_cli/src/subcommand/random_wallet.rs @@ -1,7 +1,7 @@ use crate::error::Result; use city_common::cli::user_args::RandomWalletArgs; use city_crypto::hash::qhashout::QHashOut; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; use serde::{Deserialize, Serialize}; @@ -12,7 +12,7 @@ struct RandomWalletOutputJSON { } pub async fn run(_: RandomWalletArgs) -> Result<()> { let private_key = QHashOut::::rand(); - let mut debug_wallet = DebugScenarioWallet::::new_fast_setup(); + let mut debug_wallet = CityMemoryWallet::::new_fast_setup(); let public_key = debug_wallet.add_zk_private_key(private_key); diff --git a/city_rollup_user_cli/src/subcommand/register_user.rs b/city_rollup_user_cli/src/subcommand/register_user.rs index 8182dfaa..0b71143e 100644 --- a/city_rollup_user_cli/src/subcommand/register_user.rs +++ b/city_rollup_user_cli/src/subcommand/register_user.rs @@ -1,7 +1,7 @@ use city_common::cli::user_args::RegisterUserArgs; use city_crypto::hash::qhashout::QHashOut; use city_rollup_common::api::data::block::rpc_request::CityRegisterUserRPCRequest; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_rpc_provider::{CityRpcProvider, RpcProvider}; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ fn parse_multi_hash_string(hash_str: &str) -> anyhow::Result], ) -> anyhow::Result>> { - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); Ok(private_keys .iter() .map(|private_key| wallet.add_zk_private_key(*private_key)) diff --git a/city_rollup_user_cli/src/subcommand/repl.rs b/city_rollup_user_cli/src/subcommand/repl.rs index 64ac535d..b3ef84c3 100644 --- a/city_rollup_user_cli/src/subcommand/repl.rs +++ b/city_rollup_user_cli/src/subcommand/repl.rs @@ -11,7 +11,7 @@ use city_rollup_common::{ link_api::BTCLinkAPI, traits::{QBitcoinAPIFunderSync, QBitcoinAPISync}, tx::{send_entire_balance_simple_p2pkh, send_p2pkh_exact_value}, - }, + }, qworker::job_id::QProvingJobDataIDSerializedWrapped, }; use city_rollup_rpc_provider::{CityRpcProviderSync, RpcProviderSync}; @@ -25,6 +25,17 @@ const D: usize = 2; type C = PoseidonGoldilocksConfig; type F = GoldilocksField; */ + +fn parse_job_id_array(input_str: &str) -> anyhow::Result>{ + input_str + .replace('\"', "") + .replace(' ', "") + .replace('[', "") + .replace(']', "") + .split(",") + .map(|x| QProvingJobDataIDSerializedWrapped::from_hex_string(x).map_err(|err| anyhow::anyhow!(err))) + .collect::>>() +} const MAX_CHECKPOINT_ID: u64 = 0xffffffff; struct ReplContext { pub city_rpc: RpcProviderSync, @@ -189,6 +200,27 @@ fn get_user_ids_for_public_key( Ok(Some(serde_json::to_string(&results)?)) } +fn get_proof_store_kv( + args: HashMap, + context: &mut ReplContext, +) -> Result> { + let key_str: String = args["keys"].convert()?; + let key_str = key_str.trim(); + println!("got keys: '{}'", key_str); + + if key_str.len() == 48 { + let key = QProvingJobDataIDSerializedWrapped::from_hex_string(&key_str)?; + let result = context.city_rpc.get_proof_store_value_sync(key)?; + Ok(Some(hex::encode(&result.0))) + + + + }else{ + let keys: Vec = parse_job_id_array(key_str)?; + let results = context.city_rpc.get_proof_store_values_sync(&keys)?; + Ok(Some(serde_json::to_string_pretty(&results)?)) + } +} fn spend_all(args: HashMap, context: &mut ReplContext) -> Result> { let private_key: String = args["private_key"].convert()?; @@ -376,6 +408,11 @@ pub async fn run(args: RPCReplArgs) -> Result<()> { Command::new("random_l1_wallet", random_dogecoin_wallet) .with_help("generate a random dogecoin P2PKH wallet"), ) + .add_command( + Command::new("get_proof_store_kv", get_proof_store_kv) + .with_help("get one or more keys from the proof store") + .with_parameter(Parameter::new("keys").set_required(true)?)?, + ) .add_command(Command::new("exit", exit_repl).with_help("exits the repl")) .add_command(Command::new("quit", exit_repl).with_help("exits the repl")); repl.run().map_err(|err| err.into()) diff --git a/city_rollup_user_cli/src/subcommand/token_transfer.rs b/city_rollup_user_cli/src/subcommand/token_transfer.rs index 527355dd..59661c5b 100644 --- a/city_rollup_user_cli/src/subcommand/token_transfer.rs +++ b/city_rollup_user_cli/src/subcommand/token_transfer.rs @@ -6,7 +6,7 @@ use city_common::cli::user_args::TokenTransferArgs; use city_crypto::hash::qhashout::QHashOut; use city_rollup_common::introspection::rollup::constants::get_network_magic_for_str; -use city_rollup_core_orchestrator::debug::scenario::wallet::DebugScenarioWallet; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; use city_rollup_rpc_provider::{CityRpcProvider, RpcProvider}; use plonky2::{field::goldilocks_field::GoldilocksField, plonk::config::PoseidonGoldilocksConfig}; @@ -23,7 +23,7 @@ pub async fn run(args: TokenTransferArgs) -> Result<()> { let private_key = QHashOut::::from_str(&args.private_key) .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; - let mut wallet = DebugScenarioWallet::::new_fast_setup(); + let mut wallet = CityMemoryWallet::::new_fast_setup(); let public_key = wallet.add_zk_private_key(private_key); diff --git a/city_rollup_user_prover_api/Cargo.toml b/city_rollup_user_prover_api/Cargo.toml new file mode 100644 index 00000000..f073eea1 --- /dev/null +++ b/city_rollup_user_prover_api/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "city_rollup_user_prover_api" +version = "0.1.0" +edition = "2021" + +[dependencies] +city_crypto = { path = "../city_crypto" } +city_common_circuit = { path = "../city_common_circuit" } +city_common = { path = "../city_common" } +city_rollup_common = { path = "../city_rollup_common" } +city_macros = { path = "../city_macros" } +city_rollup_circuit = { path = "../city_rollup_circuit" } + +plonky2 = { workspace = true } +bincode = { workspace = true } +jsonrpsee = { workspace = true } +tokio = { workspace = true } +anyhow = { workspace = true } +redb = { workspace = true } +hex = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +tower = { workspace = true } +tower-http = { workspace = true } +hyper = { workspace = true } + diff --git a/city_rollup_user_prover_api/src/api/mod.rs b/city_rollup_user_prover_api/src/api/mod.rs new file mode 100644 index 00000000..9f400a71 --- /dev/null +++ b/city_rollup_user_prover_api/src/api/mod.rs @@ -0,0 +1,171 @@ +use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex}; + +use city_common::data::u8bytes::{U8Bytes, U8BytesFixed}; +use city_crypto::hash::base_types::hash256::Hash256; +use city_crypto::signature::secp256k1::core::QEDCompressedSecp256K1Signature; +use jsonrpsee::core::async_trait; +use jsonrpsee::proc_macros::rpc; +use jsonrpsee::types::{ErrorObject, ErrorObjectOwned}; + +use crate::common::request::{ + UPWEncryptedPublicKeyJobRequestPayload, UPWEncryptedZKSignatureJobRequestPayload, UPWJobRequest, UPWJobRequestPayload, UPWZKSignatureJobRequestPayload +}; +use crate::worker::store::UserProverWorkerStore; + +#[rpc(server, client, namespace = "cr")] +pub trait Rpc { + #[method(name = "ping")] + async fn ping(&self, message: String) -> Result; + #[method(name = "prove_secp256k1_signature")] + async fn prove_secp256k1_signature( + &self, + public_key: U8BytesFixed<33>, + signature: U8BytesFixed<64>, + message: Hash256, + ) -> Result; + #[method(name = "prove_zk_signature")] + async fn prove_zk_signature( + &self, + private_key: Hash256, + message: Hash256, + ) -> Result; + #[method(name = "prove_zk_signature_enc")] + async fn prove_zk_signature_enc( + &self, + encrypted_private_key: Hash256, + message: Hash256, + salt: Hash256, + ) -> Result; + #[method(name = "get_zk_public_key")] + async fn get_zk_public_key( + &self, + private_key: Hash256, + ) -> Result; + #[method(name = "get_zk_public_key_enc")] + async fn get_zk_public_key_enc( + &self, + encrypted_private_key: Hash256, + salt: Hash256, + ) -> Result; + #[method(name = "get_result")] + async fn get_result(&self, id: Hash256) -> Result; +} + +#[derive(Clone)] +pub struct RpcServerImpl { + pub store: Arc>, + pub tx_worker: Sender, +} + +#[async_trait] +impl RpcServer for RpcServerImpl { + async fn ping(&self, message: String) -> Result { + Ok(message.chars().rev().collect::()) + } + async fn get_result(&self, id: Hash256) -> Result { + let result = self.store.lock().unwrap().get_result_and_clear(&id); + + if result.is_none() { + return Err(ErrorObject::owned(404, "Result not found", Some(0))); + } + Ok(U8Bytes(result.unwrap())) + } + async fn prove_secp256k1_signature( + &self, + public_key: U8BytesFixed<33>, + signature: U8BytesFixed<64>, + message: Hash256, + ) -> Result { + let id = Hash256::rand(); + let request = UPWJobRequest { + request_id: id, + payload: UPWJobRequestPayload::Secp256K1SignatureProof( + QEDCompressedSecp256K1Signature { + public_key: public_key.0, + signature: signature.0, + message, + }, + ), + }; + self.tx_worker + .send(request) + .map_err(|_| ErrorObject::owned(500, "Error sending request to worker", Some(0)))?; + Ok(id) + } + async fn prove_zk_signature( + &self, + private_key: Hash256, + message: Hash256, + ) -> Result { + let id = Hash256::rand(); + let request = UPWJobRequest { + request_id: id, + payload: UPWJobRequestPayload::ZKSignatureProof(UPWZKSignatureJobRequestPayload { + private_key, + message, + }), + }; + self.tx_worker + .send(request) + .map_err(|_| ErrorObject::owned(500, "Error sending request to worker", Some(0)))?; + Ok(id) + } + async fn prove_zk_signature_enc( + &self, + encrypted_private_key: Hash256, + message: Hash256, + salt: Hash256, + ) -> Result { + let id = Hash256::rand(); + let request = UPWJobRequest { + request_id: id, + payload: UPWJobRequestPayload::EncryptedZKSignatureProof( + UPWEncryptedZKSignatureJobRequestPayload { + message, + salt, + encrypted_private_key, + }, + ), + }; + self.tx_worker + .send(request) + .map_err(|_| ErrorObject::owned(500, "Error sending request to worker", Some(0)))?; + Ok(id) + } + + async fn get_zk_public_key( + &self, + private_key: Hash256, + ) -> Result{ + let id = Hash256::rand(); + let request = UPWJobRequest { + request_id: id, + payload: UPWJobRequestPayload::GetPublicKey(private_key), + }; + self.tx_worker + .send(request) + .map_err(|_| ErrorObject::owned(500, "Error sending request to worker", Some(0)))?; + Ok(id) + } + async fn get_zk_public_key_enc( + &self, + encrypted_private_key: Hash256, + salt: Hash256, + ) -> Result { + let id = Hash256::rand(); + let request = UPWJobRequest { + request_id: id, + payload: UPWJobRequestPayload::EncryptedGetPublicKey( + UPWEncryptedPublicKeyJobRequestPayload { + salt, + encrypted_private_key, + }, + ), + }; + self.tx_worker + .send(request) + .map_err(|_| ErrorObject::owned(500, "Error sending request to worker", Some(0)))?; + Ok(id) + } +} diff --git a/city_rollup_user_prover_api/src/common/enc.rs b/city_rollup_user_prover_api/src/common/enc.rs new file mode 100644 index 00000000..ea7f9b78 --- /dev/null +++ b/city_rollup_user_prover_api/src/common/enc.rs @@ -0,0 +1,43 @@ +use city_crypto::hash::{base_types::hash256::Hash256, core::sha256::CoreSha256Hasher}; + +pub trait SimpleEncryptionHelper: Clone + Send + Sync { + fn encrypt_32(&self, salt: Hash256, data: Hash256) -> Hash256; + fn decrypt_32(&self, salt: Hash256, encrypted_data: Hash256) -> Hash256; +} + +#[derive(Clone)] +pub struct SimpleZeroPadEncryptionHelper { + key: Hash256, +} + +impl SimpleZeroPadEncryptionHelper { + pub fn new(key: Hash256) -> Self { + Self { key } + } + pub fn new_rand() -> Self { + Self { key: Hash256::rand() } + } + pub fn new_no_encrypt() -> Self { + Self { key: Hash256::ZERO } + } + pub fn get_decryption_key(&self) -> Hash256 { + self.key + } +} + +impl SimpleEncryptionHelper for SimpleZeroPadEncryptionHelper { + fn encrypt_32(&self, salt: Hash256, data: Hash256) -> Hash256 { + let mut hasher = CoreSha256Hasher::new(); + hasher.update(&self.key.0); + hasher.update(&salt.0); + let key = hasher.finalize(); + data ^ key + } + fn decrypt_32(&self, salt: Hash256, encrypted_data: Hash256) -> Hash256 { + let mut hasher = CoreSha256Hasher::new(); + hasher.update(&self.key.0); + hasher.update(&salt.0); + let key = hasher.finalize(); + encrypted_data ^ key + } +} \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/common/mod.rs b/city_rollup_user_prover_api/src/common/mod.rs new file mode 100644 index 00000000..6b61afb9 --- /dev/null +++ b/city_rollup_user_prover_api/src/common/mod.rs @@ -0,0 +1,2 @@ +pub mod request; +pub mod enc; \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/common/request.rs b/city_rollup_user_prover_api/src/common/request.rs new file mode 100644 index 00000000..6f984281 --- /dev/null +++ b/city_rollup_user_prover_api/src/common/request.rs @@ -0,0 +1,52 @@ +use city_crypto::{hash::base_types::hash256::Hash256, signature::secp256k1::core::QEDCompressedSecp256K1Signature}; +use serde::{Serialize, Deserialize}; + +use super::enc::SimpleEncryptionHelper; + + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Copy)] +pub struct UPWEncryptedZKSignatureJobRequestPayload { + pub encrypted_private_key: Hash256, + pub salt: Hash256, + pub message: Hash256, +} +impl UPWEncryptedZKSignatureJobRequestPayload { + pub fn decrypt(&self, encryption_helper: &E) -> UPWZKSignatureJobRequestPayload { + let decrypted_key = encryption_helper.decrypt_32(self.salt, self.encrypted_private_key); + UPWZKSignatureJobRequestPayload { + private_key: decrypted_key, + message: self.message, + } + } +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Copy)] +pub struct UPWZKSignatureJobRequestPayload { + pub private_key: Hash256, + pub message: Hash256, +} +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Copy)] +pub struct UPWEncryptedPublicKeyJobRequestPayload { + pub encrypted_private_key: Hash256, + pub salt: Hash256, +} +impl UPWEncryptedPublicKeyJobRequestPayload { + pub fn decrypt(&self, encryption_helper: &E) -> Hash256 { + encryption_helper.decrypt_32(self.salt, self.encrypted_private_key) + } +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] +pub enum UPWJobRequestPayload { + Secp256K1SignatureProof(QEDCompressedSecp256K1Signature), + ZKSignatureProof(UPWZKSignatureJobRequestPayload), + EncryptedZKSignatureProof(UPWEncryptedZKSignatureJobRequestPayload), + GetPublicKey(Hash256), + EncryptedGetPublicKey(UPWEncryptedPublicKeyJobRequestPayload), +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] +pub struct UPWJobRequest { + pub request_id: Hash256, + pub payload: UPWJobRequestPayload, +} \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/lib.rs b/city_rollup_user_prover_api/src/lib.rs new file mode 100644 index 00000000..1021e5ec --- /dev/null +++ b/city_rollup_user_prover_api/src/lib.rs @@ -0,0 +1,5 @@ + +pub mod api; +pub mod worker; +pub mod common; +pub mod run; \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/run.rs b/city_rollup_user_prover_api/src/run.rs new file mode 100644 index 00000000..9195daed --- /dev/null +++ b/city_rollup_user_prover_api/src/run.rs @@ -0,0 +1,49 @@ +use std::sync::{Arc, Mutex}; + +use crate::api::RpcServer; +use crate::api::RpcServerImpl; +use crate::common::enc::SimpleZeroPadEncryptionHelper; +use crate::worker::processor::UserProverWorker; +use crate::worker::store::UserProverWorkerStore; +use city_common::logging::trace_timer::TraceTimer; +use city_crypto::hash::base_types::hash256::Hash256; +use jsonrpsee::server::Server; + +use hyper::Method; +use tower_http::cors::{Any, CorsLayer}; + +pub async fn run_server(server_addr: String, api_key: Hash256) -> anyhow::Result<()> { + let mut timer = TraceTimer::new("user_prover_api_server"); + timer.lap("initializing server"); + let encryption_helper = SimpleZeroPadEncryptionHelper::new(api_key); + let server_addr_copy = server_addr.clone(); + + let cors = CorsLayer::new() + // Allow `POST` when accessing the resource + .allow_methods([Method::POST]) + // Allow requests from any origin + .allow_origin(Any) + .allow_headers([hyper::header::CONTENT_TYPE]); + let middleware = tower::ServiceBuilder::new().layer(cors); + + // The RPC exposes the access control for filtering and the middleware for + // modifying requests / responses. These features are independent of one another + // and can also be used separately. + // In this example, we use both features. + let server = Server::builder() + .set_http_middleware(middleware) + .build(server_addr) + .await?; + let store = Arc::new(Mutex::new(UserProverWorkerStore::new())); + let tx_worker = UserProverWorker::start_worker(store.clone(), &encryption_helper); + + let store_rpc = store.clone(); + let rpc_server_impl = RpcServerImpl { + store: store_rpc, + tx_worker, + }; + timer.event(format!("server started at {}", server_addr_copy)); + let handle = server.start(rpc_server_impl.into_rpc()); + tokio::spawn(handle.stopped()); + Ok(futures::future::pending::<()>().await) +} diff --git a/city_rollup_user_prover_api/src/worker/mod.rs b/city_rollup_user_prover_api/src/worker/mod.rs new file mode 100644 index 00000000..9e4dd223 --- /dev/null +++ b/city_rollup_user_prover_api/src/worker/mod.rs @@ -0,0 +1,3 @@ +pub mod processor; +pub mod store; +pub mod prover; \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/worker/processor.rs b/city_rollup_user_prover_api/src/worker/processor.rs new file mode 100644 index 00000000..7443e3ae --- /dev/null +++ b/city_rollup_user_prover_api/src/worker/processor.rs @@ -0,0 +1,44 @@ +use std::sync::{mpsc::{self, Receiver, Sender}, Arc, Mutex}; + + + +use crate::common::{enc::SimpleEncryptionHelper, request::UPWJobRequest}; + +use super::{prover::UPWProver, store::UserProverWorkerStore}; + + + +pub struct UserProverWorker { + store: Arc>, + rx: Receiver, + encryption_helper: E, + prover: UPWProver, +} + +impl UserProverWorker { + pub fn new(store: Arc>, rx: Receiver, encryption_helper: E) -> Self { + let prover = UPWProver::new(); + Self { store, rx, encryption_helper, prover } + } + fn run_worker(&self) { + for request in self.rx.iter() { + println!("processing request: {}", request.request_id.to_hex_string()); + let result = self.prover.prove_request::(&self.encryption_helper, &request); + if result.is_ok() { + self.store.lock().unwrap().set_result(request.request_id, result.unwrap()); + println!("processed request: {}", request.request_id.to_hex_string()); + }else{ + println!("error processing request: {}, ({:?})", request.request_id.to_hex_string(), result.err()); + } + } + } + pub fn start_worker(store: Arc>, encryption_helper: &E) -> Sender { + let (tx, rx) = mpsc::channel::(); + let helper = encryption_helper.clone(); + let worker = UserProverWorker::new(store.clone(), rx, helper); + std::thread::spawn(move || { + worker.run_worker(); + }); + tx + } +} diff --git a/city_rollup_user_prover_api/src/worker/prover.rs b/city_rollup_user_prover_api/src/worker/prover.rs new file mode 100644 index 00000000..56a4025c --- /dev/null +++ b/city_rollup_user_prover_api/src/worker/prover.rs @@ -0,0 +1,46 @@ +use city_crypto::{hash::{base_types::hash256::Hash256, qhashout::QHashOut}, signature::secp256k1::core::QEDCompressedSecp256K1Signature}; +use city_rollup_circuit::wallet::memory::CityMemoryWallet; +use plonky2::{field::goldilocks_field::GoldilocksField, plonk::{config::PoseidonGoldilocksConfig, proof::ProofWithPublicInputs}}; + +use crate::common::{enc::SimpleEncryptionHelper, request::{UPWJobRequest, UPWJobRequestPayload}}; + +type F = GoldilocksField; +type C = PoseidonGoldilocksConfig; +const D: usize = 2; + +pub struct UPWProver { + wallet: CityMemoryWallet, +} + +impl UPWProver { + pub fn new() -> Self { + Self { + wallet: CityMemoryWallet::::new(), + } + } + pub fn prove_secp256k1_signature(&self, signature: &QEDCompressedSecp256K1Signature) -> anyhow::Result> { + self.wallet.zk_secp256k1_from_signature(signature) + } + pub fn prove_zk_signature(&self, private_key: QHashOut, action_hash: QHashOut) -> anyhow::Result> { + self.wallet.zk_sign_with_private_key(private_key, action_hash) + } + pub fn get_public_key_for_private_key(&self, private_key: Hash256) -> anyhow::Result> { + Ok(self.wallet.zk_wallet.basic_wallet.get_fingerprint_public_key_for_private_key(QHashOut::from_hash256_le(private_key)).to_le_bytes().to_vec()) + } + + pub fn prove_request(&self, encryption_helper: &E, request: &UPWJobRequest) -> anyhow::Result> { + match request.payload { + UPWJobRequestPayload::Secp256K1SignatureProof(core) => bincode::serialize(&self.prove_secp256k1_signature(&core)?).map_err(|e| e.into()), + UPWJobRequestPayload::ZKSignatureProof(payload) => bincode::serialize(&self.prove_zk_signature(QHashOut::from_hash256_le(payload.private_key), QHashOut::from_hash256_le(payload.message))?).map_err(|e| e.into()), + UPWJobRequestPayload::EncryptedZKSignatureProof(payload) => { + let decrypted_payload = payload.decrypt(encryption_helper); + bincode::serialize(&self.prove_zk_signature(QHashOut::from_hash256_le(decrypted_payload.private_key), QHashOut::from_hash256_le(decrypted_payload.message))?).map_err(|e| e.into()) + } + UPWJobRequestPayload::GetPublicKey(payload) => self.get_public_key_for_private_key(payload), + UPWJobRequestPayload::EncryptedGetPublicKey(payload) => { + let decrypted_payload = payload.decrypt(encryption_helper); + self.get_public_key_for_private_key(decrypted_payload) + }, + } + } +} \ No newline at end of file diff --git a/city_rollup_user_prover_api/src/worker/store.rs b/city_rollup_user_prover_api/src/worker/store.rs new file mode 100644 index 00000000..18e0486d --- /dev/null +++ b/city_rollup_user_prover_api/src/worker/store.rs @@ -0,0 +1,25 @@ +use std::collections::HashMap; + +use city_crypto::hash::base_types::hash256::Hash256; + + +#[derive(Clone, Debug)] +pub struct UserProverWorkerStore { + pub results: HashMap>, +} +impl UserProverWorkerStore { + pub fn new() -> Self { + Self { + results: HashMap::new(), + } + } + pub fn get_result(&self, key: &Hash256) -> Option<&Vec> { + self.results.get(key) + } + pub fn get_result_and_clear(&mut self, key: &Hash256) -> Option> { + self.results.remove(key) + } + pub fn set_result(&mut self, key: Hash256, value: Vec) { + self.results.insert(key, value); + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c3116b4f..2d6e0893 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ services: testing_net: ipv4_address: 172.16.238.11 restart: always - profiles: [lite,full] redis: image: redis @@ -20,7 +19,6 @@ services: testing_net: ipv4_address: 172.16.238.13 restart: always - profiles: [lite,full] networks: testing_net: ipam: diff --git a/kvq/src/traits.rs b/kvq/src/traits.rs index 752ef41d..47444bdd 100644 --- a/kvq/src/traits.rs +++ b/kvq/src/traits.rs @@ -1,13 +1,11 @@ use serde::Deserialize; use serde::Serialize; -use serde_with::serde_as; pub struct KVQPair { pub key: K, pub value: V, } -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Clone)] pub struct KVQPairSerializable { pub key: K, diff --git a/qbench_data/example.bin b/qbench_data/example.bin new file mode 100644 index 00000000..f4465230 Binary files /dev/null and b/qbench_data/example.bin differ