diff --git a/Cargo.lock b/Cargo.lock index c2e9bce94f7a0..2b8ce14d84f3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14369,6 +14369,83 @@ dependencies = [ "tracing", ] +[[package]] +name = "sui-mvr-indexer" +version = "1.37.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.5", + "backoff", + "bb8", + "bcs", + "bytes", + "cached", + "chrono", + "clap", + "criterion", + "csv", + "dashmap", + "diesel", + "diesel-async", + "diesel_migrations", + "fastcrypto", + "futures", + "hex", + "indicatif", + "itertools 0.13.0", + "jsonrpsee", + "move-binary-format", + "move-bytecode-utils", + "move-core-types", + "mysten-metrics", + "ntest", + "object_store", + "prometheus", + "rand 0.8.5", + "rayon", + "regex", + "serde", + "serde_json", + "serde_with 3.9.0", + "simulacrum", + "strum 0.24.1", + "strum_macros 0.24.3", + "sui-archival", + "sui-config", + "sui-core", + "sui-data-ingestion-core", + "sui-json", + "sui-json-rpc", + "sui-json-rpc-api", + "sui-json-rpc-types", + "sui-keys", + "sui-move-build", + "sui-open-rpc", + "sui-package-resolver", + "sui-protocol-config", + "sui-rest-api", + "sui-sdk", + "sui-snapshot", + "sui-storage", + "sui-swarm-config", + "sui-synthetic-ingestion", + "sui-test-transaction-builder", + "sui-transaction-builder", + "sui-types", + "tap", + "telemetry-subscribers", + "tempfile", + "test-cluster", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util 0.7.10", + "toml 0.7.4", + "tracing", + "url", +] + [[package]] name = "sui-network" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index a737c2360d242..28ca38cbb54a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ members = [ "crates/sui-move", "crates/sui-move-build", "crates/sui-move-lsp", + "crates/sui-mvr-indexer", "crates/sui-network", "crates/sui-node", "crates/sui-open-rpc", @@ -649,6 +650,7 @@ sui-metric-checker = { path = "crates/sui-metric-checker" } sui-move = { path = "crates/sui-move" } sui-move-build = { path = "crates/sui-move-build" } sui-move-lsp = { path = "crates/sui-move-lsp" } +sui-mvr-indexer = { path = "crates/sui-mvr-indexer" } sui-network = { path = "crates/sui-network" } sui-node = { path = "crates/sui-node" } sui-open-rpc = { path = "crates/sui-open-rpc" } diff --git a/crates/sui-mvr-indexer/Cargo.toml b/crates/sui-mvr-indexer/Cargo.toml index 0b22f81ff2bba..fd869ae522ddf 100644 --- a/crates/sui-mvr-indexer/Cargo.toml +++ b/crates/sui-mvr-indexer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sui-indexer" +name = "sui-mvr-indexer" version.workspace = true authors = ["Mysten Labs "] license = "Apache-2.0" @@ -85,5 +85,5 @@ ntest.workspace = true criterion.workspace = true [[bin]] -name = "sui-indexer" +name = "sui-mvr-indexer" path = "src/main.rs" diff --git a/crates/sui-mvr-indexer/README.md b/crates/sui-mvr-indexer/README.md index 73593434dc2fa..e579bc76ac3ad 100644 --- a/crates/sui-mvr-indexer/README.md +++ b/crates/sui-mvr-indexer/README.md @@ -1,24 +1,7 @@ -Sui indexer is an off-fullnode service to serve data from Sui protocol, including both data directly generated from chain and derivative data. +The MVR indexer is a spin-off of the Sui indexer. It has a subset of the full indexer schema, limited to just the tables needed to support MVR. The required tables are `epochs`, `checkpoints`, `packages`, `objects_snapshot`, and `objects_history`. This enables the custom indexer to support the `package_by_name` and `type_by_name` queries on GraphQL. -⚠ **Warning:** Sui indexer is still experimental and we expect occasional breaking changes that require backfills. - -## Architecture -![enhanced_FN](https://user-images.githubusercontent.com/106119108/221022505-a1d873c6-60e2-45f1-b2aa-e50192c4dfbb.png) - -## Steps to run locally -### Prerequisites -- install local [Postgres server](https://www.postgresql.org/download/). You can also `brew install postgresql@15` and then add the following to your `~/.zshrc` or `~/.zprofile`, etc: -```sh -export LDFLAGS="-L/opt/homebrew/opt/postgresql@15/lib" -export CPPFLAGS="-I/opt/homebrew/opt/postgresql@15/include" -export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH" -``` -- make sure you have libpq installed: `brew install libpq`, and in your profile, add `export PATH="/opt/homebrew/opt/libpq/bin:$PATH"`. If this doesn't work, try `brew link --force libpq`. - -- install Diesel CLI with `cargo install diesel_cli --no-default-features --features postgres`, refer to [Diesel Getting Started guide](https://diesel.rs/guides/getting-started) for more details -- [optional but handy] Postgres client like [Postico](https://eggerapps.at/postico2/), for local check, query execution etc. - -### Start the Postgres Service +# Running this indexer +## Start the Postgres Service Postgres must run as a service in the background for other tools to communicate with. If it was installed using homebrew, it can be started as a service with: @@ -26,107 +9,18 @@ Postgres must run as a service in the background for other tools to communicate brew services start postgresql@version ``` -### Local Development(Recommended) - -See the [docs](https://docs.sui.io/guides/developer/getting-started/local-network) for detailed information. Below is a quick start guide: - -Start a local network using the `sui` binary: -```sh -cargo run --bin sui -- start --with-faucet --force-regenesis -``` - -If you want to run a local network with the indexer enabled (note that `libpq` is required), you can run the following command after following the steps in the next section to set up an indexer DB: -```sh -cargo run --bin sui -- start --with-faucet --force-regenesis --with-indexer --pg-port 5432 --pg-db-name sui_indexer_v2 -``` - -### Running standalone indexer -1. DB setup, under `sui/crates/sui-indexer` run: -```sh -# an example DATABASE_URL is "postgres://postgres:postgres@localhost/exampledb" -diesel setup --database-url="" -diesel database reset --database-url="" -``` -Note that you need an existing database for this to work. Using the DATABASE_URL example in the comment of the previous code, replace `exampledb` with the name of your database. - -2. Checkout to your target branch - -For example, if you want to be on the DevNet branch -```sh -git fetch upstream devnet && git reset --hard upstream/devnet -``` -3. Start indexer binary, under `sui/crates/sui-indexer` run: -- run indexer as a writer, which pulls data from fullnode and writes data to DB -```sh -# Change the RPC_CLIENT_URL to http://0.0.0.0:9000 to run indexer against local validator & fullnode -cargo run --bin sui-indexer -- --db-url "" --rpc-client-url "https://fullnode.devnet.sui.io:443" --fullnode-sync-worker --reset-db -``` -- run indexer as a reader, which is a JSON RPC server with the [interface](https://docs.sui.io/sui-api-ref#suix_getallbalances) -``` -cargo run --bin sui-indexer -- --db-url "" --rpc-client-url "https://fullnode.devnet.sui.io:443" --rpc-server-worker -``` -More flags info can be found in this [file](src/main.rs#L41). - -### DB reset +## DB reset When making db-related changes, you may find yourself having to run migrations and reset dbs often. The commands below are how you can invoke these actions. ```sh -cargo run --bin sui-indexer -- --database-url "" reset-database --force -``` - -## Steps to run locally (TiDB) - -### Prerequisites - -1. Install TiDB - -``` sh -curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh -``` - -2. Install a compatible version of MySQL (At the time of writing, this is MySQL 8.0 -- note that 8.3 is incompatible). - -``` sh -brew install mysql@8.0 -``` - -3. Install a version of `diesel_cli` that supports MySQL (and probably also Postgres). This version of the CLI needs to be built against the version of MySQL that was installed in the previous step (compatible with the local installation of TiDB, 8.0.37 at time of writing). - -``` sh -MYSQLCLIENT_LIB_DIR=/opt/homebrew/Cellar/mysql@8.0/8.0.37/lib/ cargo install diesel_cli --no-default-features --features postgres --features mysql --force -``` - -### Run the indexer - -1.Run TiDB - -```sh -tiup playground -``` - -2.Verify tidb is running by connecting to it using the mysql client, create database `test` - -```sh -mysql --comments --host 127.0.0.1 --port 4000 -u root -create database test; +cargo run --bin sui-mvr-indexer -- --database-url "" reset-database --force ``` -3.DB setup, under `sui/crates/sui-indexer` run: - -```sh -# an example DATABASE_URL is "mysql://root:password@127.0.0.1:4000/test" -diesel setup --database-url="" --migration-dir='migrations/mysql' -diesel database reset --database-url="" --migration-dir='migrations/mysql' -``` - -Note that you need an existing database for this to work. Using the DATABASE_URL example in the comment of the previous code, replace `test` with the name of your database. -4. Run indexer as a writer, which pulls data from fullnode and writes data to DB - -```sh -# Change the RPC_CLIENT_URL to http://0.0.0.0:9000 to run indexer against local validator & fullnode -cargo run --bin sui-indexer --features mysql-feature --no-default-features -- --db-url "" --rpc-client-url "https://fullnode.devnet.sui.io:443" --fullnode-sync-worker --reset-db +## Start the indexer +```SH +cargo run --bin sui-mvr-indexer -- --db-url "" indexer --rpc-client-url "https://fullnode.devnet.sui.io:443" --remote-store-url http://lax-suifn-t99eb.devnet.sui.io:9000/rest ``` -### Extending the indexer +## Migrations To add a new table, run `diesel migration generate your_table_name`, and modify the newly created `up.sql` and `down.sql` files. diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh index f214c55f7d7eb..ea883107b31ca 100644 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh +++ b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh @@ -1,6 +1,6 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -INDEXER=${INDEXER:-"sui-indexer"} +INDEXER=${INDEXER:-"sui-mvr-indexer"} DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} "$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "UPDATE events SET sender = CASE WHEN cardinality(senders) > 0 THEN senders[1] ELSE NULL END" checkpoint_sequence_number diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh index 2492b477ed592..18a0e3b9e84de 100644 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh +++ b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh @@ -1,6 +1,6 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -INDEXER=${INDEXER:-"sui-indexer"} +INDEXER=${INDEXER:-"sui-mvr-indexer"} DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} "$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO full_objects_history (object_id, object_version, serialized_object) SELECT object_id, object_version, serialized_object FROM objects_history" checkpoint_sequence_number diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh index df1212e18bbc6..da0dc0915a0b4 100644 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh +++ b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh @@ -1,7 +1,7 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -INDEXER=${INDEXER:-"sui-indexer"} +INDEXER=${INDEXER:-"sui-mvr-indexer"} DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} "$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO tx_affected_addresses SELECT tx_sequence_number, sender AS affected, sender FROM tx_senders" tx_sequence_number "$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO tx_affected_addresses SELECT tx_sequence_number, recipient AS affected, sender FROM tx_recipients" tx_sequence_number diff --git a/crates/sui-mvr-indexer/src/main.rs b/crates/sui-mvr-indexer/src/main.rs index 85782cff9689e..703ac457398a1 100644 --- a/crates/sui-mvr-indexer/src/main.rs +++ b/crates/sui-mvr-indexer/src/main.rs @@ -2,26 +2,26 @@ // SPDX-License-Identifier: Apache-2.0 use clap::Parser; -use sui_indexer::backfill::backfill_runner::BackfillRunner; -use sui_indexer::benchmark::run_indexer_benchmark; -use sui_indexer::config::{Command, UploadOptions}; -use sui_indexer::database::ConnectionPool; -use sui_indexer::db::setup_postgres::clear_database; -use sui_indexer::db::{ +use sui_mvr_indexer::backfill::backfill_runner::BackfillRunner; +use sui_mvr_indexer::benchmark::run_indexer_benchmark; +use sui_mvr_indexer::config::{Command, UploadOptions}; +use sui_mvr_indexer::database::ConnectionPool; +use sui_mvr_indexer::db::setup_postgres::clear_database; +use sui_mvr_indexer::db::{ check_db_migration_consistency, check_prunable_tables_valid, reset_database, run_migrations, }; -use sui_indexer::indexer::Indexer; -use sui_indexer::metrics::{ +use sui_mvr_indexer::indexer::Indexer; +use sui_mvr_indexer::metrics::{ spawn_connection_pool_metric_collector, start_prometheus_server, IndexerMetrics, }; -use sui_indexer::restorer::formal_snapshot::IndexerFormalSnapshotRestorer; -use sui_indexer::store::PgIndexerStore; +use sui_mvr_indexer::restorer::formal_snapshot::IndexerFormalSnapshotRestorer; +use sui_mvr_indexer::store::PgIndexerStore; use tokio_util::sync::CancellationToken; use tracing::warn; #[tokio::main] async fn main() -> anyhow::Result<()> { - let opts = sui_indexer::config::IndexerConfig::parse(); + let opts = sui_mvr_indexer::config::IndexerConfig::parse(); // NOTE: this is to print out tracing like info, warn & error. let _guard = telemetry_subscribers::TelemetryConfig::new() diff --git a/crates/sui-mvr-indexer/src/schema.patch b/crates/sui-mvr-indexer/src/schema.patch index 0caee130c400f..c935f4d862fe0 100644 --- a/crates/sui-mvr-indexer/src/schema.patch +++ b/crates/sui-mvr-indexer/src/schema.patch @@ -1,6 +1,6 @@ -diff --git a/crates/sui-indexer/src/schema.rs b/crates/sui-indexer/src/schema.rs ---- a/crates/sui-indexer/src/schema.rs -+++ b/crates/sui-indexer/src/schema.rs +diff --git a/crates/sui-mvr-indexer/src/schema.rs b/crates/sui-mvr-indexer/src/schema.rs +--- a/crates/sui-mvr-indexer/src/schema.rs ++++ b/crates/sui-mvr-indexer/src/schema.rs @@ -1 +1,3 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 diff --git a/crates/sui-mvr-indexer/tests/ingestion_tests.rs b/crates/sui-mvr-indexer/tests/ingestion_tests.rs index 2b6a31286b27a..e205dcf60fbec 100644 --- a/crates/sui-mvr-indexer/tests/ingestion_tests.rs +++ b/crates/sui-mvr-indexer/tests/ingestion_tests.rs @@ -6,21 +6,21 @@ use diesel::ExpressionMethods; use diesel::QueryDsl; use diesel_async::RunQueryDsl; use simulacrum::Simulacrum; -use sui_indexer::errors::IndexerError; -use sui_indexer::handlers::TransactionObjectChangesToCommit; -use sui_indexer::models::{ +use sui_mvr_indexer::errors::IndexerError; +use sui_mvr_indexer::handlers::TransactionObjectChangesToCommit; +use sui_mvr_indexer::models::{ checkpoints::StoredCheckpoint, objects::StoredObject, objects::StoredObjectSnapshot, transactions::StoredTransaction, }; -use sui_indexer::schema::{checkpoints, objects, objects_snapshot, transactions}; -use sui_indexer::store::indexer_store::IndexerStore; -use sui_indexer::test_utils::{ +use sui_mvr_indexer::schema::{checkpoints, objects, objects_snapshot, transactions}; +use sui_mvr_indexer::store::indexer_store::IndexerStore; +use sui_mvr_indexer::test_utils::{ set_up, set_up_with_start_and_end_checkpoints, wait_for_checkpoint, wait_for_objects_snapshot, }; -use sui_indexer::types::EventIndex; -use sui_indexer::types::IndexedDeletedObject; -use sui_indexer::types::IndexedObject; -use sui_indexer::types::TxIndex; +use sui_mvr_indexer::types::EventIndex; +use sui_mvr_indexer::types::IndexedDeletedObject; +use sui_mvr_indexer::types::IndexedObject; +use sui_mvr_indexer::types::TxIndex; use sui_types::base_types::SuiAddress; use sui_types::effects::TransactionEffectsAPI; use sui_types::gas_coin::GasCoin; diff --git a/crates/sui-mvr-indexer/tests/read_api_tests.rs b/crates/sui-mvr-indexer/tests/read_api_tests.rs index 7b855800202db..d17b431888f01 100644 --- a/crates/sui-mvr-indexer/tests/read_api_tests.rs +++ b/crates/sui-mvr-indexer/tests/read_api_tests.rs @@ -4,10 +4,10 @@ use jsonrpsee::core::RpcResult; use simulacrum::Simulacrum; use std::sync::Arc; -use sui_indexer::apis::read_api::ReadApi; -use sui_indexer::indexer_reader::IndexerReader; -use sui_indexer::test_utils::{set_up, wait_for_checkpoint}; use sui_json_rpc_api::ReadApiServer; +use sui_mvr_indexer::apis::read_api::ReadApi; +use sui_mvr_indexer::indexer_reader::IndexerReader; +use sui_mvr_indexer::test_utils::{set_up, wait_for_checkpoint}; use tempfile::tempdir; #[tokio::test] diff --git a/docker/sui-mvr-indexer/Dockerfile b/docker/sui-mvr-indexer/Dockerfile new file mode 100644 index 0000000000000..bf0f2be29e2e7 --- /dev/null +++ b/docker/sui-mvr-indexer/Dockerfile @@ -0,0 +1,37 @@ +# Build application +# +# Copy in all crates, Cargo.toml and Cargo.lock unmodified, +# and build the application. +FROM rust:1.81-bullseye AS builder +ARG PROFILE=release +ARG GIT_REVISION +ENV GIT_REVISION=$GIT_REVISION +WORKDIR "$WORKDIR/sui" + +# sui-mvr-indexer needs postgres libpq5 and ca-certificates +RUN apt update && apt install -y libpq5 ca-certificates libpq-dev postgresql + +RUN apt-get update && apt-get install -y cmake clang + +COPY Cargo.toml Cargo.lock ./ +COPY consensus consensus +COPY crates crates +COPY sui-execution sui-execution +COPY narwhal narwhal +COPY external-crates external-crates + +RUN cargo build --profile ${PROFILE} --bin sui-mvr-indexer + +# Production Image +FROM debian:bullseye-slim AS runtime +# Use jemalloc as memory allocator +RUN apt-get update && apt-get install -y libjemalloc-dev ca-certificates curl +ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so +WORKDIR "$WORKDIR/sui" +COPY --from=builder /sui/target/release/sui-mvr-indexer /usr/local/bin +RUN apt update && apt install -y libpq5 ca-certificates libpq-dev postgresql + +ARG BUILD_DATE +ARG GIT_REVISION +LABEL build-date=$BUILD_DATE +LABEL git-revision=$GIT_REVISION diff --git a/docker/sui-mvr-indexer/build.sh b/docker/sui-mvr-indexer/build.sh new file mode 100755 index 0000000000000..5e1c8c1623fe7 --- /dev/null +++ b/docker/sui-mvr-indexer/build.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# fast fail. +set -e + +DIR="$( cd "$( dirname "$0" )" && pwd )" +REPO_ROOT="$(git rev-parse --show-toplevel)" +DOCKERFILE="$DIR/Dockerfile" +GIT_REVISION="$(git describe --always --abbrev=12 --dirty --exclude '*')" +BUILD_DATE="$(date -u +'%Y-%m-%d')" + +echo +echo "Building sui-mvr-indexer docker image" +echo "Dockerfile: \t$DOCKERFILE" +echo "docker context: $REPO_ROOT" +echo "build date: \t$BUILD_DATE" +echo "git revision: \t$GIT_REVISION" +echo + +docker build -f "$DOCKERFILE" "$REPO_ROOT" \ + --build-arg GIT_REVISION="$GIT_REVISION" \ + --build-arg BUILD_DATE="$BUILD_DATE" \ + "$@"