From ee6414a0d4d812e7759e18a714895676df5b2293 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 10 May 2024 05:38:36 +0000 Subject: [PATCH] A0-4231: Extract chain bootstrap to a separate crate (#1699) # Description This final PR achieves the goal of removing the `aleph-runtime` dependency from `aleph-node`. A new crate `chain-bootstraper` is made from the current chain bootstrap logic in `aleph-node`. The dependencies in `aleph-node` should be clearer now: * `aleph-node` does not compile native runtime anymore via default features build. * `chain-bootstrapper` does compile both native and wasm runtime during build. The amount of packages that node compiles is about 150 less than it was before. Depending on the machine, It's up to 40% compilation speedup for a`aleph-node` binary. Here's local build, clean repo, on my laptop: After change - `aleph-node` and `chain-bootstrapper` builds in the similar time. ``` time ./scripts/run_nodes.sh -v 6 2024-04-22 13:37:54:233 Starting ./scripts/run_nodes.sh 2024-04-22 13:37:54:234 Creating base path ./run-nodes-local if it does not exist 2024-04-22 13:37:54:236 Stopping all current node processes 2024-04-22 13:37:54:241 No aleph-node processes found. 2024-04-22 13:37:54:242 Building testing aleph-node binary (short session) and chainspec-generator binary. Compiling proc-macro2 v1.0.81 [...] Finished release [optimized] target(s) in 9m 02s [...] Finished release [optimized] target(s) in 9m 31s [...] ________________________________________________________ Executed in 18.59 mins fish external usr time 119.30 mins 874.00 micros 119.30 mins sys time 5.67 mins 0.00 micros 5.67 mins ``` Before change - note though the overall time is faster than before: ``` marcin@marcin-Latitude-3520 ~/g/aleph-node-3 (main)> time ./scripts/run_nodes.sh -v 6 (base) [...] Finished release [optimized] target(s) in 16m 44s [...] Executed in 16.77 mins fish external usr time 107.24 mins 0.00 micros 107.24 mins sys time 5.32 mins 791.00 micros 5.32 mins ``` This is because of `polkadot-sdk` crates architecture - we still need to compile most of the crates in both node and bootstrapper. There's small optimization in this PR to not build three polkadot-sdk crates with default features, ie to disable `rocks-db` build when `chain-bootstrapper` is built. In Ci though, compilation should be faster as we can parallelize `aleph-node` and `chain-bootstrapper` build, which this PR achieves so. Most of the changes are workflows anyway. ## Type of change Please delete options that are not relevant. - Breaking change (fix or feature that would cause existing functionality to not work as expected) # Checklist: * Deploy to devnet: - will be tested after merge to `main`. * Nightly feature-gated builds: https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8828449304 * Nightly pipeline e2e tests on featurenet: - FEs engine require small change, and will be tested after merge to `main`. * Nightly pipeline integration tests: https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8828450804 * Nightly pipeline logic e2e tests: https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8828452203 * Nightly pipeline normal session e2e tests: https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8828453925 * On main push workflow (edited so it's no-op): https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/8828493822 --- .dockerignore | 2 + .github/actions/run-e2e-test/action.yml | 15 +++- .github/scripts/run_consensus.sh | 14 +++- .github/scripts/test_python_general.sh | 20 ++++- .../_build-and-push-featurenet-node-image.yml | 8 +- .../workflows/_build-chain-bootstrapper.yml | 78 +++++++++++++++++ .../_build-production-node-and-runtime.yml | 5 +- ...e-and-runtime.yml => _build-test-node.yml} | 24 +++--- .../_check-production-node-and-runtime.yml | 5 +- .github/workflows/_push-node-image-to-ecr.yml | 32 ++++--- .github/workflows/deploy-to-devnet.yml | 2 +- .../nightly-check-node-features-build.yml | 14 +++- .github/workflows/nightly-e2e-logic-tests.yml | 27 ++++-- .../nightly-normal-session-e2e-tests.yml | 84 +++++++++++++++---- .../on-main-or-release-branch-commit.yml | 23 ++++- .github/workflows/on-pull-request-commit.yml | 18 +++- BUILD.md | 4 +- Cargo.lock | 36 ++++---- Cargo.toml | 6 +- bin/chain-bootstrapper/Cargo.toml | 40 +++++++++ bin/chain-bootstrapper/Dockerfile | 6 ++ .../src/chain_spec/builder.rs | 0 .../src/chain_spec/cli.rs | 0 .../src/chain_spec/commands.rs | 3 +- .../src/chain_spec/keystore.rs | 3 +- .../src/chain_spec/mod.rs | 9 -- bin/chain-bootstrapper/src/main.rs | 32 +++++++ bin/node/Cargo.toml | 61 ++++++++------ bin/node/src/cli.rs | 23 ++--- bin/node/src/executor.rs | 4 +- bin/node/src/lib.rs | 3 +- bin/node/src/main.rs | 2 - bin/node/src/service.rs | 4 +- local-tests/chainrunner/chain.py | 8 +- local-tests/run_nodes.py | 8 +- local-tests/test_force_reorg.py | 9 +- local-tests/test_major_sync.py | 12 ++- local-tests/test_recover_after_abft_update.py | 8 +- primitives/src/lib.rs | 3 + scripts/run_nodes.sh | 21 +++-- 40 files changed, 512 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/_build-chain-bootstrapper.yml rename .github/workflows/{_build-test-node-and-runtime.yml => _build-test-node.yml} (67%) create mode 100644 bin/chain-bootstrapper/Cargo.toml create mode 100644 bin/chain-bootstrapper/Dockerfile rename bin/{node => chain-bootstrapper}/src/chain_spec/builder.rs (100%) rename bin/{node => chain-bootstrapper}/src/chain_spec/cli.rs (100%) rename bin/{node => chain-bootstrapper}/src/chain_spec/commands.rs (98%) rename bin/{node => chain-bootstrapper}/src/chain_spec/keystore.rs (98%) rename bin/{node => chain-bootstrapper}/src/chain_spec/mod.rs (75%) create mode 100644 bin/chain-bootstrapper/src/main.rs diff --git a/.dockerignore b/.dockerignore index 644ad99be7..fc9865afbe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ **/target target !target/release/aleph-node +!target/release/chain-bootstrapper +!target/production/chain-bootstrapper !target/production/aleph-node !bin/cliain/target/release/cliain diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 2fada9a7ce..7ed32bee15 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -38,6 +38,10 @@ inputs: aleph-e2e-client-image: description: 'aleph-e2e-client image' required: true + chain-bootstrapper-image: + description: 'chain-bootstrapper image' + required: false + default: chain-bootstrapper-image-test timeout-minutes: description: 'The maximum number of minutes to let a test run before it is canceled and considered failed' @@ -54,7 +58,7 @@ runs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v5 - - name: Download artifact with docker image + - name: Download node docker image uses: actions/download-artifact@v4 with: name: ${{ inputs.image-path }} @@ -63,6 +67,15 @@ runs: shell: bash run: docker load -i aleph-node.tar + - name: Download chain-bootstrapper image + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.chain-bootstrapper-image }} + + - name: Load chain-bootstrapper image + shell: bash + run: docker load -i chain-bootstrapper.tar + - name: Run consensus party shell: bash run: | diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 889fdaf42c..eb52abba26 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -20,6 +20,7 @@ MIN_VALIDATOR_COUNT=${MIN_VALIDATOR_COUNT:-4} DOCKER_COMPOSE=${DOCKER_COMPOSE:-docker/docker-compose.yml} OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""} NODE_IMAGE=${NODE_IMAGE:-"aleph-node:latest"} +CHAIN_BOOTSTRAPPER_IMAGE=${CHAIN_BOOTSTRAPPER_IMAGE:-"chain-bootstrapper:latest"} LOGS_OUTPUT_FILE=${LOGS_OUTPUT_FILE:=""} # ------------------------ argument parsing and usage ----------------------- @@ -81,8 +82,8 @@ function generate_chainspec() { local validator_ids_comma_separated="${validators//${IFS:0:1}/,}" echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}" - docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \ - -c "aleph-node bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" + docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER_IMAGE}" \ + -c "chain-bootstrapper bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" } function generate_bootnode_peer_id() { @@ -162,6 +163,15 @@ else exit 1 fi +if docker inspect ${NODE_IMAGE} > /dev/null; then + echo "aleph-node image tag ${NODE_IMAGE} found locally" +else + echo "${NODE_IMAGE} not found locally." + echo "Build image first with:" + echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ." + exit 1 +fi + mkdir -p docker/data/ echo "Warning: if you run this script on the same machine more then once, and finalization does not work, remove docker/data." global_account_ids=$(generate_account_ids ${NODE_COUNT}) diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index a7a2e0d89d..d62d517f94 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -8,8 +8,10 @@ function usage(){ cat << EOF Usage: $0 - --aleph-node BINARY] - path to aleph-node-binary + --aleph-node BINARY + path to aleph-node binary + --chain-bootstrapper BINARY + path to chain-bootstrapper binary --testcase NAME name of python file in local-tests directory to run EOF @@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; do ALEPH_NODE_BINARY="$2" shift;shift ;; + --chain-bootstrapper) + CHAIN_BOOTSTRAPPER="$2" + shift;shift + ;; --testcase) TESTCASE="$2" shift;shift @@ -40,7 +46,11 @@ done pushd local-tests/ > /dev/null if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then - echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}." + echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}" + exit 1 +fi +if [[ ! -f "${CHAIN_BOOTSTRAPPER}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}" exit 1 fi if [[ -z "${TESTCASE}" ]]; then @@ -56,6 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then fi chmod +x "${ALEPH_NODE_BINARY}" +chmod +x "${CHAIN_BOOTSTRAPPER}" echo "Installing python requirements" pip install -r requirements.txt @@ -65,6 +76,9 @@ pip install -r requirements.txt # first buffered and that you can see the output of your application. export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY +export CHAIN_BOOTSTRAPPER +export RUST_LOG=debug export WORKDIR=$(mktemp -d) +echo "WORKDIR is ${WORKDIR}" eval "./${file_name_to_run}" popd > /dev/null diff --git a/.github/workflows/_build-and-push-featurenet-node-image.yml b/.github/workflows/_build-and-push-featurenet-node-image.yml index bcabd1e62b..fe9c6fcfc0 100644 --- a/.github/workflows/_build-and-push-featurenet-node-image.yml +++ b/.github/workflows/_build-and-push-featurenet-node-image.yml @@ -17,10 +17,10 @@ on: type: boolean required: true jobs: - build-test-node-and-runtime: + build-test-node: if: ${{ inputs.short-session == true }} name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ inputs.ref }} @@ -33,8 +33,8 @@ jobs: build-and-push-featurnet-image-to-ecr: name: Build and push featurnet image - needs: [build-test-node-and-runtime, build-production-node-and-runtime] - # to prevent this job to be skipped as at least on of dependant jobs is skipped + needs: [build-test-node, build-production-node-and-runtime] + # this job should not be skipped in case one of its parent job is skipped if: ${{ !cancelled() }} runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/_build-chain-bootstrapper.yml b/.github/workflows/_build-chain-bootstrapper.yml new file mode 100644 index 0000000000..d3e5293c6c --- /dev/null +++ b/.github/workflows/_build-chain-bootstrapper.yml @@ -0,0 +1,78 @@ +--- +# This workflow builds chain-bootstrapper binary, dedicated fer generating chainspecs for +# test and dev chains +name: Build chain-bootstrapper +on: + workflow_call: + inputs: + ref: + description: 'git ref: hash, branch, tag to build chain-bootstrapper binary from' + type: string + required: true + production: + description: 'set to true to use production profile' + type: boolean + required: true + +jobs: + main: + name: Build chain-bootstrapper + runs-on: [self-hosted, Linux, X64, large] + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + CARGO_FOLDER: ${{ inputs.production == true && 'production' || 'release' }} + ARTIFACT_NAME_SUFFIX: ${{ inputs.production == true && 'release' || 'test' }} + steps: + - name: Checkout aleph-node source code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Call action get-ref-properties + id: get-ref-properties + uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 + + - name: Install Rust toolchain + uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 + with: + targets: wasm32-unknown-unknown + + - name: Build without production profile + if: ${{ inputs.production != true }} + run: | + cargo build --release -p chain-bootstrapper \ + --features "short_session enable_treasury_proposals" + + - name: Build with production profile + if: ${{ inputs.production == true }} + run: | + cargo build --profile production -p chain-bootstrapper + + - name: Upload binary to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper-${{ env.ARTIFACT_NAME_SUFFIX }} + path: target/${{ env.CARGO_FOLDER }}/chain-bootstrapper + if-no-files-found: error + retention-days: 7 + + - name: Build docker + id: build-image + run: | + chmod +x target/${{ env.CARGO_FOLDER }}/chain-bootstrapper + if [[ ${{ inputs.production }} == true ]]; then + mkdir -p target/release + mv target/production/chain-bootstrapper target/release/ + fi + docker build --tag chain-bootstrapper:latest -f ./bin/chain-bootstrapper/Dockerfile . + docker save -o chain-bootstrapper.tar chain-bootstrapper:latest + + - name: Upload docker image to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper-image-${{ env.ARTIFACT_NAME_SUFFIX }} + path: chain-bootstrapper.tar + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/_build-production-node-and-runtime.yml b/.github/workflows/_build-production-node-and-runtime.yml index e0e8ce7c2e..05159a7a3a 100644 --- a/.github/workflows/_build-production-node-and-runtime.yml +++ b/.github/workflows/_build-production-node-and-runtime.yml @@ -32,7 +32,7 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Build production binary and runtime + - name: Build production aleph-node run: cargo build --profile production -p aleph-node - name: Upload release binary to GH artifacts @@ -43,6 +43,9 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Build production aleph-runtime + run: cargo build --profile production -p aleph-runtime + # required by _check-runtime-determinism.yml workflow - name: Upload release runtime to GH artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_build-test-node-and-runtime.yml b/.github/workflows/_build-test-node.yml similarity index 67% rename from .github/workflows/_build-test-node-and-runtime.yml rename to .github/workflows/_build-test-node.yml index 0c70cf297e..876c02615b 100644 --- a/.github/workflows/_build-test-node-and-runtime.yml +++ b/.github/workflows/_build-test-node.yml @@ -1,7 +1,6 @@ --- -# This workflow builds test version of aleph-node and aleph-runtime, ie with some -# features enabled. Those binaries MUST NOT be used in the production. -name: Build test node and runtime +# This workflow builds test version of aleph-node. It MUST NOT be used in the production. +name: Build test node on: workflow_call: inputs: @@ -11,8 +10,8 @@ on: required: true jobs: - build-artifacts: - name: Build test node and runtime + main: + name: Build test node runs-on: [self-hosted, Linux, X64, large] env: RUST_BACKTRACE: full @@ -30,16 +29,13 @@ jobs: - name: Install Rust toolchain uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 - with: - targets: wasm32-unknown-unknown - - name: Build test binary and runtime + - name: Build run: | - cargo build --release -p aleph-node \ - --features "short_session enable_treasury_proposals only_legacy" + cargo build --release -p aleph-node --features only_legacy # this is required for some workflows, yet ideally it should not be required - - name: Upload test binary to GH Artifacts + - name: Upload binary to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-node @@ -47,15 +43,15 @@ jobs: if-no-files-found: error retention-days: 7 - - name: Build test docker image + - name: Build docker id: build-image run: | chmod +x target/release/aleph-node docker build --tag aleph-node:latest -f ./docker/Dockerfile . docker save -o aleph-node.tar aleph-node:latest - # this is solely required by action/run-e2e-test.yml, also it should not be required - - name: Upload test docker image + # this is required by action/run-e2e-test.yml + - name: Upload docker to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-docker diff --git a/.github/workflows/_check-production-node-and-runtime.yml b/.github/workflows/_check-production-node-and-runtime.yml index ec01b19f77..508c012eff 100644 --- a/.github/workflows/_check-production-node-and-runtime.yml +++ b/.github/workflows/_check-production-node-and-runtime.yml @@ -25,5 +25,8 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Check production binary and runtime + - name: Check aleph-node run: cargo check --profile production -p aleph-node + + - name: Check aleph-runtime + run: cargo check --profile production -p aleph-runtime diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index 8aa6f55aaf..3e1865182a 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -1,21 +1,29 @@ --- -# This workflow builds docker aleph-node image based on binary stored in GH artifacts -name: Build and push node image to ECR +# This workflow builds arbitrary docker node image based on binary downloaded in GH artifacts +name: Build and push docker image to ECR on: workflow_call: inputs: - node-binary: - description: 'Name of aleph-node binary stored in GH artifacts' + binary-artifact-name: + description: 'Name of a binary stored in GH artifacts' + required: true + type: string + binary-name: + description: 'Name of a binary to build docker image on top of' required: true type: string docker-image-name: - description: 'Name of node docker to be uploaded to ECR' + description: 'Name of node docker image to be uploaded to ECR' + required: true + type: string + docker-file-path: + description: 'Path to Dockerfile' required: true type: string jobs: main: - name: Push node docker image to the ECR + name: Push ${{ inputs.binary-name }} docker image to the ECR runs-on: ubuntu-20.04 steps: - name: Checkout aleph-node sources @@ -25,17 +33,19 @@ jobs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 - - name: Download ${{ inputs.node-binary }} binary from artifacts + - name: Download ${{ inputs.binary-artifact-name }} from artifacts uses: actions/download-artifact@v4 with: - name: ${{ inputs.node-binary }} + name: ${{ inputs.binary-artifact-name }} path: target/release/ - - name: Build docker node image + - name: Build ${{ inputs.binary-name }} docker image id: build-image run: | - chmod +x target/release/aleph-node - docker build --tag ${{ inputs.docker-image-name }}:latest -f ./docker/Dockerfile . + chmod +x target/release/${{ inputs.binary-name }} + docker build \ + --tag ${{ inputs.docker-image-name }}:latest \ + -f ${{ inputs.docker-file-path }} . - name: Login to Public Amazon ECR id: login-public-ecr diff --git a/.github/workflows/deploy-to-devnet.yml b/.github/workflows/deploy-to-devnet.yml index be63cc9459..58d841f32c 100644 --- a/.github/workflows/deploy-to-devnet.yml +++ b/.github/workflows/deploy-to-devnet.yml @@ -98,7 +98,7 @@ jobs: EVE=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint \ - '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}aleph-node:${{ env.RELEASE_TAG }}' \ + '/usr/local/bin/chain-bootstrapper' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \ bootstrap-chain --raw --base-path /data --chain-id a0dnet1 \ --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q \ --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD \ diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index a393e48e27..cb3d503c84 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -29,11 +29,21 @@ jobs: - name: aleph-runtime with try-runtime run: cargo check --profile production -p aleph-runtime --features try-runtime --locked + - name: aleph-runtime with runtime-benchmarks + # yamllint disable-line rule:line-length + run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks --locked + - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked - - name: aleph-node with local-debugging - run: cargo check --profile dev -p aleph-node --features local-debugging --locked + - name: aleph-node with try-runtime + run: cargo check --profile production -p aleph-node --features try-runtime --locked + + - name: aleph-node with aleph-runtime-native + run: cargo check --profile dev -p aleph-node --features aleph-runtime-native --locked + + - name: chain-bootstrapper with try-runtime + run: cargo check --profile production -p chain-bootstrapper --features try-runtime --locked slack-notification: name: Slack notification diff --git a/.github/workflows/nightly-e2e-logic-tests.yml b/.github/workflows/nightly-e2e-logic-tests.yml index 97a8f831b3..b000358030 100644 --- a/.github/workflows/nightly-e2e-logic-tests.yml +++ b/.github/workflows/nightly-e2e-logic-tests.yml @@ -19,12 +19,19 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit - build-test-node-and-runtime: + build-test-node: name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ github.ref }} + build-chain-bootstrapper-test: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: false + build-aleph-e2e-client-image: needs: [check-vars-and-secrets] name: Build aleph-e2e-client image @@ -36,31 +43,41 @@ jobs: run-e2e-tests: name: Run e2e tests - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node + - build-chain-bootstrapper-test + - build-aleph-e2e-client-image uses: ./.github/workflows/_run-e2e-tests.yml with: # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} run-recover-after-abft-update-test: - needs: [build-test-node-and-runtime] + needs: [build-test-node, build-chain-bootstrapper-test] name: Run recovery test after abft update runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download test node from GH artifacts uses: actions/download-artifact@v4 with: name: aleph-test-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-test + path: target/release/ + - name: Run test timeout-minutes: 15 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_recover_after_abft_update check-nightly-pipeline-completion: diff --git a/.github/workflows/nightly-normal-session-e2e-tests.yml b/.github/workflows/nightly-normal-session-e2e-tests.yml index 64e3030522..7c602e45a4 100644 --- a/.github/workflows/nightly-normal-session-e2e-tests.yml +++ b/.github/workflows/nightly-normal-session-e2e-tests.yml @@ -37,8 +37,18 @@ jobs: name: Build synthetic node uses: ./.github/workflows/_build-synthetic-node.yml + build-chain-bootstrapper-production: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: true + run-e2e-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Run high out-latency test runs-on: ubuntu-20.04 steps: @@ -51,13 +61,17 @@ jobs: test-case: high_out_latency_for_all image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 40 run-e2e-sync-test-one_node_catching_up_and_then_becoming_necessary_for_consensus: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up and then becoming necessary for consensus runs-on: ubuntu-20.04 @@ -75,6 +89,7 @@ jobs: test-case: test::sync::one_node_catching_up_and_then_becoming_necessary_for_consensus image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -82,7 +97,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-one_node_catching_up: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up runs-on: ubuntu-20.04 steps: @@ -99,6 +117,7 @@ jobs: test-case: test::sync::one_node_catching_up image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -106,7 +125,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_and_one_quorum_and_switch_quorum_between_them: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups and one quorum and switch quorum between them runs-on: ubuntu-20.04 @@ -124,6 +146,7 @@ jobs: test-case: test::sync::into_two_groups_and_one_quorum_and_switch_quorum_between_them image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -131,7 +154,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_multiple_groups_of_two: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into multiple groups of two runs-on: ubuntu-20.04 steps: @@ -148,6 +174,7 @@ jobs: test-case: test::sync::into_multiple_groups_of_two image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -155,7 +182,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_equal_size_groups_with_no_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two equal size groups with no quorum runs-on: ubuntu-20.04 @@ -173,6 +203,7 @@ jobs: test-case: test::sync::into_two_equal_size_groups_with_no_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -180,7 +211,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_one_with_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups one with quorum runs-on: ubuntu-20.04 @@ -198,6 +232,7 @@ jobs: test-case: test::sync::into_two_groups_one_with_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -205,7 +240,10 @@ jobs: timeout-minutes: 35 run-e2e-finalization-stall: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Finalization-stall test using synthetic-network runs-on: ubuntu-20.04 steps: @@ -223,56 +261,73 @@ jobs: test-case: test::sync::large_finalization_stall image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.finalization_stall_with_pruning.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 60 run-major-sync-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run major sync test runs-on: ubuntu-20.04 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-release + path: target/release/ + - name: Run test timeout-minutes: 65 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_major_sync run-force-reorg-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run force reorgs test runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-release + path: target/release/ + - name: Run test timeout-minutes: 25 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_force_reorg - run-e2e-no-quorum-without-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Run high out-latency for every quorum runs-on: ubuntu-20.04 steps: @@ -286,6 +341,7 @@ jobs: image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet compose-file: docker/docker-compose.synthetic-network.yml + chain-bootstrapper-image: chain-bootstrapper-image-release # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 35 diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 0b47ab36ad..011aaa661d 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -13,6 +13,25 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit + build-chain-bootstrapper: + needs: [check-vars-and-secrets] + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: true + + push-chain-bootstrapper-image-to-ecr: + name: Push chain-bootstrapper docker image to ECR + needs: [build-chain-bootstrapper] + uses: ./.github/workflows/_push-node-image-to-ecr.yml + secrets: inherit + with: + binary-artifact-name: chain-bootstrapper-release + docker-image-name: chain-bootstrapper + binary-name: chain-bootstrapper + docker-file-path: ./bin/chain-bootstrapper/Dockerfile + build-and-push-cliain: needs: [check-vars-and-secrets] name: Build and push cliain @@ -63,8 +82,10 @@ jobs: uses: ./.github/workflows/_push-node-image-to-ecr.yml secrets: inherit with: - node-binary: aleph-production-node + binary-artifact-name: aleph-production-node docker-image-name: aleph-node + binary-name: aleph-node + docker-file-path: ./docker/Dockerfile send-slack-notification-release: name: Send Slack notification about workflow status diff --git a/.github/workflows/on-pull-request-commit.yml b/.github/workflows/on-pull-request-commit.yml index cb91dd95f9..d4e56ca0c6 100644 --- a/.github/workflows/on-pull-request-commit.yml +++ b/.github/workflows/on-pull-request-commit.yml @@ -38,12 +38,19 @@ jobs: name: Check production node and runtime uses: ./.github/workflows/_check-production-node-and-runtime.yml - build-test-node-and-runtime: - name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + build-test-node: + name: Build test node + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ github.ref }} + build-chain-bootstrapper: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: false + build-aleph-e2e-client-image: name: Build aleph-e2e-client image uses: ./.github/workflows/_build-aleph-e2e-client-image.yml @@ -53,7 +60,10 @@ jobs: run-e2e-finalization-test: name: Run e2e finalization test - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper runs-on: ubuntu-20.04 steps: - name: Checkout source code diff --git a/BUILD.md b/BUILD.md index 1ab7fd5ec4..82bde88e8b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -106,9 +106,9 @@ After a successful build the binary can be found in `target/release/aleph-node`. ### Local debugging -If you'd like to use Rust debugger with aleph-node binary, built it with `local-debugging` feature: +If you'd like to use Rust debugger with aleph-node binary, built it with `aleph-native-runtime` feature: ``` -cargo check --profile dev -p aleph-node --features local-debugging --locked +cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked ``` [nix]: https://nixos.org/download.html diff --git a/Cargo.lock b/Cargo.lock index fdfe93e995..f0d45d3735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,24 +279,19 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "futures", - "hex", - "hex-literal", "jsonrpsee", - "libp2p", "log", - "pallet-staking", "pallet-transaction-payment-rpc", "parity-scale-codec", "primitives", "sc-basic-authorship", - "sc-chain-spec", "sc-cli", "sc-client-api", + "sc-client-db", "sc-consensus", "sc-consensus-aura", "sc-consensus-slots", "sc-executor", - "sc-keystore", "sc-network", "sc-network-sync", "sc-rpc", @@ -308,7 +303,6 @@ dependencies = [ "serde", "serde_json", "sp-api", - "sp-application-crypto", "sp-arithmetic", "sp-block-builder", "sp-blockchain", @@ -316,7 +310,6 @@ dependencies = [ "sp-consensus-aura", "sp-core", "sp-io", - "sp-keystore", "sp-runtime", "sp-timestamp", "sp-transaction-pool", @@ -1267,6 +1260,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chain-bootstrapper" +version = "0.1.0" +dependencies = [ + "aleph-runtime", + "hex", + "libp2p", + "pallet-staking", + "primitives", + "sc-chain-spec", + "sc-cli", + "sc-client-db", + "sc-keystore", + "sc-service", + "serde", + "serde_json", + "sp-application-crypto", + "sp-core", + "sp-runtime", +] + [[package]] name = "chrono" version = "0.4.38" @@ -3248,12 +3262,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - [[package]] name = "hkdf" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index 6b6d8bdfd0..c2bbfdc731 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "baby-liminal-extension", "bin/node", "bin/runtime", + "bin/chain-bootstrapper", "clique", "finality-aleph", "pallets/aleph", @@ -116,7 +117,8 @@ pallet-proxy = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.gi sc-basic-authorship = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-block-builder = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-chain-spec = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } -sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } +sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } +sc-client-db = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } sc-client-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-consensus = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-consensus-aura = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } @@ -128,7 +130,7 @@ sc-network-common = { git = "https://github.com/Cardinal-Cryptography/polkadot-s sc-network-sync = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-rpc = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-rpc-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } -sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } +sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } sc-telemetry = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-transaction-pool = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-transaction-pool-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } diff --git a/bin/chain-bootstrapper/Cargo.toml b/bin/chain-bootstrapper/Cargo.toml new file mode 100644 index 0000000000..ee17515c56 --- /dev/null +++ b/bin/chain-bootstrapper/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "chain-bootstrapper" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +primitives = { workspace = true } +aleph-runtime = { workspace = true } +pallet-staking = { workspace = true } +libp2p = { workspace = true } +hex = { workspace = true } + +sc-cli = { workspace = true, default-features = false } +sc-chain-spec = { workspace = true } +sc-service = { workspace = true, default-features = false } +sc-client-db = { workspace = true, default-features = false } +sc-keystore = { workspace = true } + +sp-application-crypto = { workspace = true } +sp-runtime = { workspace = true } +sp-core = { workspace = true } + +serde_json = { workspace = true } +serde = { workspace = true } + +[features] +default = [] +short_session = [ + "aleph-runtime/short_session", + "primitives/short_session", +] +enable_treasury_proposals = [ + "aleph-runtime/enable_treasury_proposals" +] +try-runtime = [ + "aleph-runtime/try-runtime", +] diff --git a/bin/chain-bootstrapper/Dockerfile b/bin/chain-bootstrapper/Dockerfile new file mode 100644 index 0000000000..92147703a6 --- /dev/null +++ b/bin/chain-bootstrapper/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:jammy + +COPY target/release/chain-bootstrapper /usr/local/bin +RUN chmod +x /usr/local/bin/chain-bootstrapper + +ENTRYPOINT ["chain-bootstrapper"] diff --git a/bin/node/src/chain_spec/builder.rs b/bin/chain-bootstrapper/src/chain_spec/builder.rs similarity index 100% rename from bin/node/src/chain_spec/builder.rs rename to bin/chain-bootstrapper/src/chain_spec/builder.rs diff --git a/bin/node/src/chain_spec/cli.rs b/bin/chain-bootstrapper/src/chain_spec/cli.rs similarity index 100% rename from bin/node/src/chain_spec/cli.rs rename to bin/chain-bootstrapper/src/chain_spec/cli.rs diff --git a/bin/node/src/chain_spec/commands.rs b/bin/chain-bootstrapper/src/chain_spec/commands.rs similarity index 98% rename from bin/node/src/chain_spec/commands.rs rename to bin/chain-bootstrapper/src/chain_spec/commands.rs index 3c8b20f99a..e9cf6a1485 100644 --- a/bin/node/src/chain_spec/commands.rs +++ b/bin/chain-bootstrapper/src/chain_spec/commands.rs @@ -1,5 +1,6 @@ use std::{io::Write, path::PathBuf}; +use primitives::DEFAULT_BACKUP_FOLDER; use sc_cli::{ clap::{self, Parser}, Error, KeystoreParams, @@ -12,7 +13,7 @@ use crate::chain_spec::{ keystore::{ create_abft_backup_dir, create_account_session_keys, create_p2p_key, open_keystore, }, - AlephNodeChainSpec, DEFAULT_BACKUP_FOLDER, + AlephNodeChainSpec, }; /// This command generates session keys and libp2p key for all input accounts. diff --git a/bin/node/src/chain_spec/keystore.rs b/bin/chain-bootstrapper/src/chain_spec/keystore.rs similarity index 98% rename from bin/node/src/chain_spec/keystore.rs rename to bin/chain-bootstrapper/src/chain_spec/keystore.rs index c17caa254e..dd3e54de4c 100644 --- a/bin/node/src/chain_spec/keystore.rs +++ b/bin/chain-bootstrapper/src/chain_spec/keystore.rs @@ -6,11 +6,10 @@ use std::{ use libp2p::identity::ed25519 as libp2p_ed25519; use primitives::{AccountId, AuraId, AuthorityId}; use sc_cli::KeystoreParams; -use sc_keystore::LocalKeystore; +use sc_keystore::{Keystore, LocalKeystore}; use sc_service::{config::KeystoreConfig, BasePath}; use serde::{Deserialize, Serialize}; use sp_core::crypto::key_types; -use sp_keystore::Keystore; #[derive(Clone, Deserialize, Serialize)] pub struct AccountSessionKeys { diff --git a/bin/node/src/chain_spec/mod.rs b/bin/chain-bootstrapper/src/chain_spec/mod.rs similarity index 75% rename from bin/node/src/chain_spec/mod.rs rename to bin/chain-bootstrapper/src/chain_spec/mod.rs index a49e90b0f5..ed90c5dc2b 100644 --- a/bin/node/src/chain_spec/mod.rs +++ b/bin/chain-bootstrapper/src/chain_spec/mod.rs @@ -10,7 +10,6 @@ pub const CHAINTYPE_LOCAL: &str = "local"; pub const CHAINTYPE_LIVE: &str = "live"; pub const DEFAULT_CHAIN_ID: &str = "a0dnet1"; -pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; pub const DEFAULT_SUDO_ACCOUNT_ALICE: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; @@ -20,14 +19,6 @@ use sc_chain_spec::ChainType; use sc_cli::Error; use sp_application_crypto::Ss58Codec; -pub fn mainnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::mainnet_chainspec()) -} - -pub fn testnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::testnet_chainspec()) -} - fn parse_chaintype(s: &str) -> Result { Ok(match s { CHAINTYPE_DEV => ChainType::Development, diff --git a/bin/chain-bootstrapper/src/main.rs b/bin/chain-bootstrapper/src/main.rs new file mode 100644 index 0000000000..4ba27a10a9 --- /dev/null +++ b/bin/chain-bootstrapper/src/main.rs @@ -0,0 +1,32 @@ +mod chain_spec; + +use sc_cli::clap::{self, Parser, Subcommand as ClapSubcommand}; + +use crate::chain_spec::{BootstrapChainCmd, ConvertChainspecToRawCmd}; + +#[derive(Parser)] +struct Cli { + #[command(subcommand)] + pub subcommand: Option, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, ClapSubcommand)] +pub enum Subcommand { + /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout + BootstrapChain(BootstrapChainCmd), + + /// Takes a chainspec and generates a corresponding raw chainspec + ConvertChainspecToRaw(ConvertChainspecToRawCmd), +} + +fn main() -> sc_cli::Result<()> { + let cli = Cli::parse(); + + match &cli.subcommand { + Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), + Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), + + None => Err("Command was required!".into()), + } +} diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index a416bb9e25..f50bfcb714 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -21,48 +21,36 @@ log = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } futures = { workspace = true } -hex = { workspace = true } -hex-literal = { workspace = true } -libp2p = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } -sc-basic-authorship = { workspace = true } -sc-chain-spec = { workspace = true } sc-cli = { workspace = true } +sc-basic-authorship = { workspace = true } sc-client-api = { workspace = true } sc-consensus = { workspace = true } sc-consensus-aura = { workspace = true } sc-consensus-slots = { workspace = true } sc-executor = { workspace = true } -sc-keystore = { workspace = true } sc-network = { workspace = true } sc-network-sync = { workspace = true } sc-service = { workspace = true } +sc-client-db = { workspace = true } sc-telemetry = { workspace = true } sc-transaction-pool = { workspace = true } sc-transaction-pool-api = { workspace = true } -sp-application-crypto = { workspace = true } sp-arithmetic = { workspace = true } sp-block-builder = { workspace = true } sp-consensus = { workspace = true } sp-consensus-aura = { workspace = true } sp-core = { workspace = true } sp-io = { workspace = true } -sp-keystore = { workspace = true } sp-runtime = { workspace = true } sp-timestamp = { workspace = true } sp-transaction-pool = { workspace = true } -pallet-staking = { workspace = true } -try-runtime-cli = { workspace = true, optional = true } - frame-benchmarking-cli = { workspace = true, optional = true } frame-benchmarking = { workspace = true, optional = true } -# this is only neeeded for chainspec generation -aleph-runtime = { workspace = true } - aleph-runtime-interfaces = { workspace = true } finality-aleph = { workspace = true } primitives = { workspace = true } @@ -77,28 +65,47 @@ sp-blockchain = { workspace = true } substrate-frame-rpc-system = { workspace = true } pallet-transaction-payment-rpc = { workspace = true } +aleph-runtime = { workspace = true, optional = true } +try-runtime-cli = { workspace = true, optional = true } + [build-dependencies] substrate-build-script-utils = { workspace = true } [features] -default = [] -short_session = [ - "aleph-runtime/short_session", - "primitives/short_session", -] -try-runtime = [ - "aleph-runtime/try-runtime", - "try-runtime-cli/try-runtime", -] -enable_treasury_proposals = [ - "aleph-runtime/enable_treasury_proposals" +default = ["std", "rocksdb"] +std = [ + "parity-scale-codec/std", + "sp-api/std", + "sp-block-builder/std", + "sp-consensus-aura/std", + "sp-core/std", + "sp-runtime/std", + "sp-transaction-pool/std", + "primitives/std", + "sp-io/std", + "aleph-runtime-interfaces/std", ] runtime-benchmarks = [ + "aleph-runtime-native", "frame-benchmarking", "frame-benchmarking-cli/runtime-benchmarks", - "aleph-runtime/runtime-benchmarks", + "aleph-runtime?/runtime-benchmarks", ] +rocksdb = [ + "sc-cli/rocksdb", + "sc-client-db/rocksdb", + "sc-service/rocksdb", +] +try-runtime = [ + "aleph-runtime-native", + "dep:try-runtime-cli", + "aleph-runtime?/try-runtime", + "try-runtime-cli?/try-runtime", +] +aleph-runtime-native = [ + "dep:aleph-runtime" +] + only_legacy = [ "finality-aleph/only_legacy" ] -local-debugging = [] diff --git a/bin/node/src/cli.rs b/bin/node/src/cli.rs index 565cc9b30a..e2c7b811e8 100644 --- a/bin/node/src/cli.rs +++ b/bin/node/src/cli.rs @@ -3,12 +3,11 @@ use sc_cli::{ PurgeChainCmd, RunCmd, SubstrateCli, }; +pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; + use crate::{ aleph_cli::AlephCli, - chain_spec, - chain_spec::{ - commands::BootstrapChainCmd, mainnet_config, testnet_config, ConvertChainspecToRawCmd, - }, + resources::{mainnet_chainspec, testnet_chainspec}, }; #[derive(Debug, Parser)] @@ -24,6 +23,14 @@ pub struct Cli { pub run: RunCmd, } +pub fn mainnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(mainnet_chainspec()) +} + +pub fn testnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(testnet_chainspec()) +} + impl SubstrateCli for Cli { fn impl_name() -> String { "Aleph Node".into() @@ -58,7 +65,7 @@ impl SubstrateCli for Cli { "mainnet" => mainnet_config(), "testnet" => testnet_config(), - _ => chain_spec::AlephNodeChainSpec::from_json_file(id.into()), + _ => AlephNodeChainSpec::from_json_file(id.into()), }; Ok(Box::new(chainspec?)) } @@ -70,12 +77,6 @@ pub enum Subcommand { #[command(subcommand)] Key(sc_cli::KeySubcommand), - /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout - BootstrapChain(BootstrapChainCmd), - - /// Takes a chainspec and generates a corresponding raw chainspec - ConvertChainspecToRaw(ConvertChainspecToRawCmd), - /// Validate blocks. CheckBlock(sc_cli::CheckBlockCmd), diff --git a/bin/node/src/executor.rs b/bin/node/src/executor.rs index 749e94c41f..71693a3755 100644 --- a/bin/node/src/executor.rs +++ b/bin/node/src/executor.rs @@ -6,7 +6,7 @@ use sc_service::Configuration; #[cfg(not(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" )))] pub mod aleph_executor { @@ -27,7 +27,7 @@ pub mod aleph_executor { #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub mod aleph_executor { diff --git a/bin/node/src/lib.rs b/bin/node/src/lib.rs index bb2731cce3..dc2bf4c39e 100644 --- a/bin/node/src/lib.rs +++ b/bin/node/src/lib.rs @@ -1,6 +1,5 @@ mod aleph_cli; mod aleph_node_rpc; -mod chain_spec; mod cli; mod config; mod executor; @@ -12,7 +11,7 @@ pub use cli::{Cli, Subcommand}; pub use config::Validator as ConfigValidator; #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub use executor::aleph_executor::ExecutorDispatch; diff --git a/bin/node/src/main.rs b/bin/node/src/main.rs index 0a7a7cc54f..7d681c1c9a 100644 --- a/bin/node/src/main.rs +++ b/bin/node/src/main.rs @@ -15,8 +15,6 @@ fn main() -> sc_cli::Result<()> { let config_validation_result = ConfigValidator::process(&mut cli); match &cli.subcommand { - Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), - Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::CheckBlock(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/bin/node/src/service.rs b/bin/node/src/service.rs index 978a0cedce..0a3ca0f4ee 100644 --- a/bin/node/src/service.rs +++ b/bin/node/src/service.rs @@ -13,7 +13,8 @@ use finality_aleph::{ }; use log::warn; use primitives::{ - fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, MAX_BLOCK_SIZE, + fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, DEFAULT_BACKUP_FOLDER, + MAX_BLOCK_SIZE, }; use sc_basic_authorship::ProposerFactory; use sc_client_api::{BlockBackend, HeaderBackend}; @@ -29,7 +30,6 @@ use sp_consensus_aura::{sr25519::AuthorityPair as AuraPair, Slot}; use crate::{ aleph_cli::AlephCli, - chain_spec::DEFAULT_BACKUP_FOLDER, executor::aleph_executor, rpc::{create_full as create_full_rpc, FullDeps as RpcFullDeps}, }; diff --git a/local-tests/chainrunner/chain.py b/local-tests/chainrunner/chain.py index e3b66eebec..548dc6a93a 100644 --- a/local-tests/chainrunner/chain.py +++ b/local-tests/chainrunner/chain.py @@ -43,16 +43,16 @@ def new(self, binary, validators, nonvalidators=None): self.nodes = self.validator_nodes + self.nonvalidator_nodes - def bootstrap(self, binary, validators, nonvalidators=None, raw=True, **kwargs): + def bootstrap(self, aleph_node_binary, chain_bootstrapper_binary, validators, nonvalidators=None, raw=True, **kwargs): """Bootstrap the chain. `validators` and `nonvalidators` should be lists of strings with public keys. Flags `--account-ids`, `--base-path` and `--raw` are added automatically. All other flags are taken from kwargs""" - self.new(binary, validators, nonvalidators) + self.new(aleph_node_binary, validators, nonvalidators) nonvalidators = nonvalidators or [] all_accounts = list(validators) + list(nonvalidators) - binary = check_file(binary) - cmd = [binary, 'bootstrap-chain', + binary = check_file(chain_bootstrapper_binary) + cmd = [chain_bootstrapper_binary, 'bootstrap-chain', '--base-path', self.path, '--account-ids', ','.join(all_accounts), '--authorities-account-ids', ','.join(validators)] diff --git a/local-tests/run_nodes.py b/local-tests/run_nodes.py index e801ed2fc7..fff35abd92 100755 --- a/local-tests/run_nodes.py +++ b/local-tests/run_nodes.py @@ -9,19 +9,21 @@ nodes = 4 workdir = '.' -binary = '../target/release/aleph-node' +aleph_node_binary = '../target/release/aleph-node' +chain_bootstrapper_binary = '../target/release/chain-bootstrapper' port = 30334 rpc_port = 9944 phrases = ['//Alice', '//Bob', '//Charlie', '//Dave', '//Ezekiel', '//Fanny', '//George', '//Hugo'] -keys_dict = generate_keys(binary, phrases) +keys_dict = generate_keys(aleph_node_binary, phrases) keys = list(keys_dict.values()) nodes = min(nodes, len(phrases)) chain = Chain(workdir) print(f'Bootstrapping chain for {nodes} nodes') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys[:nodes], chain_type='local') chain.set_flags('validator', diff --git a/local-tests/test_force_reorg.py b/local-tests/test_force_reorg.py index eb9ee1cb50..90da775088 100755 --- a/local-tests/test_force_reorg.py +++ b/local-tests/test_force_reorg.py @@ -12,14 +12,17 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary (important DON'T use short-session feature): -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) + phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_major_sync.py b/local-tests/test_major_sync.py index 561d665d49..f9d1c29a8a 100755 --- a/local-tests/test_major_sync.py +++ b/local-tests/test_major_sync.py @@ -12,15 +12,19 @@ workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) logging.info(f"Workdir is {workdir}") -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) -logging.info(f"aleph-node binary is {binary}") +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) +logging.info(f"aleph-node binary is {aleph_node_binary}") +logging.info(f"chain_bootstrapper binary is {chain_bootstrapper_binary}") + phrases = [f'//{i}' for i in range(5)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_recover_after_abft_update.py b/local-tests/test_recover_after_abft_update.py index 8f40586160..8b8926204d 100755 --- a/local-tests/test_recover_after_abft_update.py +++ b/local-tests/test_recover_after_abft_update.py @@ -17,14 +17,16 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary. -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index c102864fb6..ee9b60a14d 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -159,6 +159,9 @@ pub const LENIENT_THRESHOLD: Perquintill = Perquintill::from_percent(90); /// Number of non-finalized blocks that halts block production pub const DEFAULT_MAX_NON_FINALIZED_BLOCKS: u32 = 20; +/// A relative folder where to store ABFT backups +pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; + /// Hold set of validators that produce blocks and set of validators that participate in finality /// during session. #[derive(Decode, Encode, TypeInfo, Debug, Clone, PartialEq, Eq)] diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index bb38ce3107..cb98f9bf7b 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -39,7 +39,8 @@ set -euo pipefail # ------------------------ constants -------------------------------------- -export ALEPH_NODE="target/release/aleph-node" +ALEPH_NODE="target/release/aleph-node" +CHAINSPEC_GENERATOR="target/release/chain-bootstrapper" NODE_P2P_PORT_RANGE_START=30333 NODE_VALIDATOR_PORT_RANGE_START=30343 NODE_RPC_PORT_RANGE_START=9944 @@ -193,8 +194,11 @@ function run_node() { # ------------------------- input checks ---------------------------------- +if [[ "${VALIDATORS}" -lt 1 ]]; then + error "Number of validators should be at least 1!" +fi if [[ "${VALIDATORS}" -lt 4 ]]; then - error "Number of validators should be at least 4!" + warning "AlephBFT is only supported for more than 4 nodes." fi if [[ "${RPC_NODES}" -lt 1 ]]; then error "Number of RPC nodes should be at least 1!" @@ -227,10 +231,13 @@ if ! killall -9 aleph-node 2> /dev/null; then fi if [[ -z "${DONT_BUILD_ALEPH_NODE}" ]]; then - info "Building testing aleph-node binary (short session)." - cargo build --release -p aleph-node --features "short_session enable_treasury_proposals" -elif [[ ! -x "${ALEPH_NODE}" ]]; then - error "${ALEPH_NODE} does not exist or it's not an executable file!" + info "Building testing aleph-node binary (short session) and chain-bootstrapper binary." + cargo build --release -p aleph-node + if [[ -z "${DONT_BOOTSTRAP}" ]]; then + cargo build --release -p chain-bootstrapper --features "short_session enable_treasury_proposals" + fi +elif [[ ! -x "${ALEPH_NODE}" || ! -x "${CHAINSPEC_GENERATOR}" ]]; then + error "${ALEPH_NODE} or ${CHAINSPEC_GENERATOR} does not exist or it's not an executable file!" fi NUMBER_OF_NODES_TO_BOOTSTRAP=$(( VALIDATORS + RPC_NODES )) @@ -263,7 +270,7 @@ if [[ -z "${DONT_BOOTSTRAP}" ]]; then validator_ids_string="${validator_ids_string//${IFS:0:1}/,}" info "Populating keystore for all accounts with session keys and libp2p key, and generating chainspec" - "${ALEPH_NODE}" bootstrap-chain \ + "${CHAINSPEC_GENERATOR}" bootstrap-chain \ --raw \ --base-path "${BASE_PATH}" \ --account-ids "${all_account_ids_string}" \