diff --git a/.dockerignore b/.dockerignore index 644ad99be7..fc9865afbe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ **/target target !target/release/aleph-node +!target/release/chain-bootstrapper +!target/production/chain-bootstrapper !target/production/aleph-node !bin/cliain/target/release/cliain diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 2fada9a7ce..7ed32bee15 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -38,6 +38,10 @@ inputs: aleph-e2e-client-image: description: 'aleph-e2e-client image' required: true + chain-bootstrapper-image: + description: 'chain-bootstrapper image' + required: false + default: chain-bootstrapper-image-test timeout-minutes: description: 'The maximum number of minutes to let a test run before it is canceled and considered failed' @@ -54,7 +58,7 @@ runs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v5 - - name: Download artifact with docker image + - name: Download node docker image uses: actions/download-artifact@v4 with: name: ${{ inputs.image-path }} @@ -63,6 +67,15 @@ runs: shell: bash run: docker load -i aleph-node.tar + - name: Download chain-bootstrapper image + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.chain-bootstrapper-image }} + + - name: Load chain-bootstrapper image + shell: bash + run: docker load -i chain-bootstrapper.tar + - name: Run consensus party shell: bash run: | diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index 889fdaf42c..eb52abba26 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -20,6 +20,7 @@ MIN_VALIDATOR_COUNT=${MIN_VALIDATOR_COUNT:-4} DOCKER_COMPOSE=${DOCKER_COMPOSE:-docker/docker-compose.yml} OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""} NODE_IMAGE=${NODE_IMAGE:-"aleph-node:latest"} +CHAIN_BOOTSTRAPPER_IMAGE=${CHAIN_BOOTSTRAPPER_IMAGE:-"chain-bootstrapper:latest"} LOGS_OUTPUT_FILE=${LOGS_OUTPUT_FILE:=""} # ------------------------ argument parsing and usage ----------------------- @@ -81,8 +82,8 @@ function generate_chainspec() { local validator_ids_comma_separated="${validators//${IFS:0:1}/,}" echo "Generate chainspec and keystores for accounts: ${account_ids_comma_separated[@]}" - docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${NODE_IMAGE}" \ - -c "aleph-node bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" + docker run --rm -v $(pwd)/docker/data:/data --entrypoint "/bin/sh" -e RUST_LOG=debug "${CHAIN_BOOTSTRAPPER_IMAGE}" \ + -c "chain-bootstrapper bootstrap-chain --base-path /data --account-ids ${account_ids_comma_separated} --authorities-account-ids ${validator_ids_comma_separated} > /data/chainspec.json" } function generate_bootnode_peer_id() { @@ -162,6 +163,15 @@ else exit 1 fi +if docker inspect ${NODE_IMAGE} > /dev/null; then + echo "aleph-node image tag ${NODE_IMAGE} found locally" +else + echo "${NODE_IMAGE} not found locally." + echo "Build image first with:" + echo "docker build -t ${NODE_IMAGE} -f docker/Dockerfile ." + exit 1 +fi + mkdir -p docker/data/ echo "Warning: if you run this script on the same machine more then once, and finalization does not work, remove docker/data." global_account_ids=$(generate_account_ids ${NODE_COUNT}) diff --git a/.github/scripts/test_python_general.sh b/.github/scripts/test_python_general.sh index a7a2e0d89d..d62d517f94 100755 --- a/.github/scripts/test_python_general.sh +++ b/.github/scripts/test_python_general.sh @@ -8,8 +8,10 @@ function usage(){ cat << EOF Usage: $0 - --aleph-node BINARY] - path to aleph-node-binary + --aleph-node BINARY + path to aleph-node binary + --chain-bootstrapper BINARY + path to chain-bootstrapper binary --testcase NAME name of python file in local-tests directory to run EOF @@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; do ALEPH_NODE_BINARY="$2" shift;shift ;; + --chain-bootstrapper) + CHAIN_BOOTSTRAPPER="$2" + shift;shift + ;; --testcase) TESTCASE="$2" shift;shift @@ -40,7 +46,11 @@ done pushd local-tests/ > /dev/null if [[ ! -f "${ALEPH_NODE_BINARY}" ]]; then - echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}." + echo "Error: aleph-node binary does not exist at given path ${ALEPH_NODE_BINARY}" + exit 1 +fi +if [[ ! -f "${CHAIN_BOOTSTRAPPER}" ]]; then + echo "Error: chain-bootstrapper binary does not exist at given path ${CHAIN_BOOTSTRAPPER}" exit 1 fi if [[ -z "${TESTCASE}" ]]; then @@ -56,6 +66,7 @@ if [[ ! -x "${file_name_to_run}" ]]; then fi chmod +x "${ALEPH_NODE_BINARY}" +chmod +x "${CHAIN_BOOTSTRAPPER}" echo "Installing python requirements" pip install -r requirements.txt @@ -65,6 +76,9 @@ pip install -r requirements.txt # first buffered and that you can see the output of your application. export PYTHONUNBUFFERED=y export ALEPH_NODE_BINARY +export CHAIN_BOOTSTRAPPER +export RUST_LOG=debug export WORKDIR=$(mktemp -d) +echo "WORKDIR is ${WORKDIR}" eval "./${file_name_to_run}" popd > /dev/null diff --git a/.github/workflows/_build-and-push-featurenet-node-image.yml b/.github/workflows/_build-and-push-featurenet-node-image.yml index bcabd1e62b..fe9c6fcfc0 100644 --- a/.github/workflows/_build-and-push-featurenet-node-image.yml +++ b/.github/workflows/_build-and-push-featurenet-node-image.yml @@ -17,10 +17,10 @@ on: type: boolean required: true jobs: - build-test-node-and-runtime: + build-test-node: if: ${{ inputs.short-session == true }} name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ inputs.ref }} @@ -33,8 +33,8 @@ jobs: build-and-push-featurnet-image-to-ecr: name: Build and push featurnet image - needs: [build-test-node-and-runtime, build-production-node-and-runtime] - # to prevent this job to be skipped as at least on of dependant jobs is skipped + needs: [build-test-node, build-production-node-and-runtime] + # this job should not be skipped in case one of its parent job is skipped if: ${{ !cancelled() }} runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/_build-chain-bootstrapper.yml b/.github/workflows/_build-chain-bootstrapper.yml new file mode 100644 index 0000000000..d3e5293c6c --- /dev/null +++ b/.github/workflows/_build-chain-bootstrapper.yml @@ -0,0 +1,78 @@ +--- +# This workflow builds chain-bootstrapper binary, dedicated fer generating chainspecs for +# test and dev chains +name: Build chain-bootstrapper +on: + workflow_call: + inputs: + ref: + description: 'git ref: hash, branch, tag to build chain-bootstrapper binary from' + type: string + required: true + production: + description: 'set to true to use production profile' + type: boolean + required: true + +jobs: + main: + name: Build chain-bootstrapper + runs-on: [self-hosted, Linux, X64, large] + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + CARGO_FOLDER: ${{ inputs.production == true && 'production' || 'release' }} + ARTIFACT_NAME_SUFFIX: ${{ inputs.production == true && 'release' || 'test' }} + steps: + - name: Checkout aleph-node source code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Call action get-ref-properties + id: get-ref-properties + uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 + + - name: Install Rust toolchain + uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 + with: + targets: wasm32-unknown-unknown + + - name: Build without production profile + if: ${{ inputs.production != true }} + run: | + cargo build --release -p chain-bootstrapper \ + --features "short_session enable_treasury_proposals" + + - name: Build with production profile + if: ${{ inputs.production == true }} + run: | + cargo build --profile production -p chain-bootstrapper + + - name: Upload binary to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper-${{ env.ARTIFACT_NAME_SUFFIX }} + path: target/${{ env.CARGO_FOLDER }}/chain-bootstrapper + if-no-files-found: error + retention-days: 7 + + - name: Build docker + id: build-image + run: | + chmod +x target/${{ env.CARGO_FOLDER }}/chain-bootstrapper + if [[ ${{ inputs.production }} == true ]]; then + mkdir -p target/release + mv target/production/chain-bootstrapper target/release/ + fi + docker build --tag chain-bootstrapper:latest -f ./bin/chain-bootstrapper/Dockerfile . + docker save -o chain-bootstrapper.tar chain-bootstrapper:latest + + - name: Upload docker image to GH Artifacts + uses: actions/upload-artifact@v4 + with: + name: chain-bootstrapper-image-${{ env.ARTIFACT_NAME_SUFFIX }} + path: chain-bootstrapper.tar + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/_build-production-node-and-runtime.yml b/.github/workflows/_build-production-node-and-runtime.yml index e0e8ce7c2e..05159a7a3a 100644 --- a/.github/workflows/_build-production-node-and-runtime.yml +++ b/.github/workflows/_build-production-node-and-runtime.yml @@ -32,7 +32,7 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Build production binary and runtime + - name: Build production aleph-node run: cargo build --profile production -p aleph-node - name: Upload release binary to GH artifacts @@ -43,6 +43,9 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Build production aleph-runtime + run: cargo build --profile production -p aleph-runtime + # required by _check-runtime-determinism.yml workflow - name: Upload release runtime to GH artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_build-test-node-and-runtime.yml b/.github/workflows/_build-test-node.yml similarity index 67% rename from .github/workflows/_build-test-node-and-runtime.yml rename to .github/workflows/_build-test-node.yml index 0c70cf297e..876c02615b 100644 --- a/.github/workflows/_build-test-node-and-runtime.yml +++ b/.github/workflows/_build-test-node.yml @@ -1,7 +1,6 @@ --- -# This workflow builds test version of aleph-node and aleph-runtime, ie with some -# features enabled. Those binaries MUST NOT be used in the production. -name: Build test node and runtime +# This workflow builds test version of aleph-node. It MUST NOT be used in the production. +name: Build test node on: workflow_call: inputs: @@ -11,8 +10,8 @@ on: required: true jobs: - build-artifacts: - name: Build test node and runtime + main: + name: Build test node runs-on: [self-hosted, Linux, X64, large] env: RUST_BACKTRACE: full @@ -30,16 +29,13 @@ jobs: - name: Install Rust toolchain uses: Cardinal-Cryptography/github-actions/install-rust-toolchain@v6 - with: - targets: wasm32-unknown-unknown - - name: Build test binary and runtime + - name: Build run: | - cargo build --release -p aleph-node \ - --features "short_session enable_treasury_proposals only_legacy" + cargo build --release -p aleph-node --features only_legacy # this is required for some workflows, yet ideally it should not be required - - name: Upload test binary to GH Artifacts + - name: Upload binary to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-node @@ -47,15 +43,15 @@ jobs: if-no-files-found: error retention-days: 7 - - name: Build test docker image + - name: Build docker id: build-image run: | chmod +x target/release/aleph-node docker build --tag aleph-node:latest -f ./docker/Dockerfile . docker save -o aleph-node.tar aleph-node:latest - # this is solely required by action/run-e2e-test.yml, also it should not be required - - name: Upload test docker image + # this is required by action/run-e2e-test.yml + - name: Upload docker to GH Artifacts uses: actions/upload-artifact@v4 with: name: aleph-test-docker diff --git a/.github/workflows/_check-production-node-and-runtime.yml b/.github/workflows/_check-production-node-and-runtime.yml index ec01b19f77..508c012eff 100644 --- a/.github/workflows/_check-production-node-and-runtime.yml +++ b/.github/workflows/_check-production-node-and-runtime.yml @@ -25,5 +25,8 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Check production binary and runtime + - name: Check aleph-node run: cargo check --profile production -p aleph-node + + - name: Check aleph-runtime + run: cargo check --profile production -p aleph-runtime diff --git a/.github/workflows/_push-node-image-to-ecr.yml b/.github/workflows/_push-node-image-to-ecr.yml index 8aa6f55aaf..3e1865182a 100644 --- a/.github/workflows/_push-node-image-to-ecr.yml +++ b/.github/workflows/_push-node-image-to-ecr.yml @@ -1,21 +1,29 @@ --- -# This workflow builds docker aleph-node image based on binary stored in GH artifacts -name: Build and push node image to ECR +# This workflow builds arbitrary docker node image based on binary downloaded in GH artifacts +name: Build and push docker image to ECR on: workflow_call: inputs: - node-binary: - description: 'Name of aleph-node binary stored in GH artifacts' + binary-artifact-name: + description: 'Name of a binary stored in GH artifacts' + required: true + type: string + binary-name: + description: 'Name of a binary to build docker image on top of' required: true type: string docker-image-name: - description: 'Name of node docker to be uploaded to ECR' + description: 'Name of node docker image to be uploaded to ECR' + required: true + type: string + docker-file-path: + description: 'Path to Dockerfile' required: true type: string jobs: main: - name: Push node docker image to the ECR + name: Push ${{ inputs.binary-name }} docker image to the ECR runs-on: ubuntu-20.04 steps: - name: Checkout aleph-node sources @@ -25,17 +33,19 @@ jobs: id: get-ref-properties uses: Cardinal-Cryptography/github-actions/get-ref-properties@v6 - - name: Download ${{ inputs.node-binary }} binary from artifacts + - name: Download ${{ inputs.binary-artifact-name }} from artifacts uses: actions/download-artifact@v4 with: - name: ${{ inputs.node-binary }} + name: ${{ inputs.binary-artifact-name }} path: target/release/ - - name: Build docker node image + - name: Build ${{ inputs.binary-name }} docker image id: build-image run: | - chmod +x target/release/aleph-node - docker build --tag ${{ inputs.docker-image-name }}:latest -f ./docker/Dockerfile . + chmod +x target/release/${{ inputs.binary-name }} + docker build \ + --tag ${{ inputs.docker-image-name }}:latest \ + -f ${{ inputs.docker-file-path }} . - name: Login to Public Amazon ECR id: login-public-ecr diff --git a/.github/workflows/deploy-to-devnet.yml b/.github/workflows/deploy-to-devnet.yml index be63cc9459..58d841f32c 100644 --- a/.github/workflows/deploy-to-devnet.yml +++ b/.github/workflows/deploy-to-devnet.yml @@ -98,7 +98,7 @@ jobs: EVE=5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw docker run -i -v $(pwd)/data:/data --env RUST_BACKTRACE=1 --entrypoint \ - '/usr/local/bin/aleph-node' '${{ vars.ECR_PUBLIC_REGISTRY }}aleph-node:${{ env.RELEASE_TAG }}' \ + '/usr/local/bin/chain-bootstrapper' '${{ vars.ECR_PUBLIC_REGISTRY }}chain-bootstrapper:${{ env.RELEASE_TAG }}' \ bootstrap-chain --raw --base-path /data --chain-id a0dnet1 \ --account-ids 5D34dL5prEUaGNQtPPZ3yN5Y6BnkfXunKXXz6fo7ZJbLwRRH,5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o,5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9,5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK,5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW,5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR,5DZLHESsfGrJ5YzT3HuRPXsSNb589xQ4Unubh1mYLodzKdVY,5GHJzqvG6tXnngCpG7B12qjUvbo5e4e9z8Xjidk3CQZHxTPZ,5CUnSsgAyLND3bxxnfNhgWXSe9Wn676JzLpGLgyJv858qhoX,5CVKn7HAZW1Ky4r7Vkgsr7VEW88C2sHgUNDiwHY9Ct2hjU8q \ --sudo-account-id 5F4SvwaUEQubiqkPF8YnRfcN77cLsT2DfG4vFeQmSXNjR7hD \ diff --git a/.github/workflows/nightly-check-node-features-build.yml b/.github/workflows/nightly-check-node-features-build.yml index a393e48e27..cb3d503c84 100644 --- a/.github/workflows/nightly-check-node-features-build.yml +++ b/.github/workflows/nightly-check-node-features-build.yml @@ -29,11 +29,21 @@ jobs: - name: aleph-runtime with try-runtime run: cargo check --profile production -p aleph-runtime --features try-runtime --locked + - name: aleph-runtime with runtime-benchmarks + # yamllint disable-line rule:line-length + run: cargo check --profile production -p aleph-runtime --features runtime-benchmarks --locked + - name: aleph-node with runtime-benchmarks run: cargo check --profile production -p aleph-node --features runtime-benchmarks --locked - - name: aleph-node with local-debugging - run: cargo check --profile dev -p aleph-node --features local-debugging --locked + - name: aleph-node with try-runtime + run: cargo check --profile production -p aleph-node --features try-runtime --locked + + - name: aleph-node with aleph-runtime-native + run: cargo check --profile dev -p aleph-node --features aleph-runtime-native --locked + + - name: chain-bootstrapper with try-runtime + run: cargo check --profile production -p chain-bootstrapper --features try-runtime --locked slack-notification: name: Slack notification diff --git a/.github/workflows/nightly-e2e-logic-tests.yml b/.github/workflows/nightly-e2e-logic-tests.yml index 97a8f831b3..b000358030 100644 --- a/.github/workflows/nightly-e2e-logic-tests.yml +++ b/.github/workflows/nightly-e2e-logic-tests.yml @@ -19,12 +19,19 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit - build-test-node-and-runtime: + build-test-node: name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ github.ref }} + build-chain-bootstrapper-test: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: false + build-aleph-e2e-client-image: needs: [check-vars-and-secrets] name: Build aleph-e2e-client image @@ -36,31 +43,41 @@ jobs: run-e2e-tests: name: Run e2e tests - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node + - build-chain-bootstrapper-test + - build-aleph-e2e-client-image uses: ./.github/workflows/_run-e2e-tests.yml with: # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} run-recover-after-abft-update-test: - needs: [build-test-node-and-runtime] + needs: [build-test-node, build-chain-bootstrapper-test] name: Run recovery test after abft update runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download test node from GH artifacts uses: actions/download-artifact@v4 with: name: aleph-test-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-test + path: target/release/ + - name: Run test timeout-minutes: 15 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_recover_after_abft_update check-nightly-pipeline-completion: diff --git a/.github/workflows/nightly-normal-session-e2e-tests.yml b/.github/workflows/nightly-normal-session-e2e-tests.yml index 64e3030522..7c602e45a4 100644 --- a/.github/workflows/nightly-normal-session-e2e-tests.yml +++ b/.github/workflows/nightly-normal-session-e2e-tests.yml @@ -37,8 +37,18 @@ jobs: name: Build synthetic node uses: ./.github/workflows/_build-synthetic-node.yml + build-chain-bootstrapper-production: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: true + run-e2e-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Run high out-latency test runs-on: ubuntu-20.04 steps: @@ -51,13 +61,17 @@ jobs: test-case: high_out_latency_for_all image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 40 run-e2e-sync-test-one_node_catching_up_and_then_becoming_necessary_for_consensus: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up and then becoming necessary for consensus runs-on: ubuntu-20.04 @@ -75,6 +89,7 @@ jobs: test-case: test::sync::one_node_catching_up_and_then_becoming_necessary_for_consensus image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -82,7 +97,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-one_node_catching_up: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - one node catching up runs-on: ubuntu-20.04 steps: @@ -99,6 +117,7 @@ jobs: test-case: test::sync::one_node_catching_up image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -106,7 +125,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_and_one_quorum_and_switch_quorum_between_them: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups and one quorum and switch quorum between them runs-on: ubuntu-20.04 @@ -124,6 +146,7 @@ jobs: test-case: test::sync::into_two_groups_and_one_quorum_and_switch_quorum_between_them image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -131,7 +154,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_multiple_groups_of_two: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into multiple groups of two runs-on: ubuntu-20.04 steps: @@ -148,6 +174,7 @@ jobs: test-case: test::sync::into_multiple_groups_of_two image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -155,7 +182,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_equal_size_groups_with_no_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two equal size groups with no quorum runs-on: ubuntu-20.04 @@ -173,6 +203,7 @@ jobs: test-case: test::sync::into_two_equal_size_groups_with_no_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -180,7 +211,10 @@ jobs: timeout-minutes: 35 run-e2e-sync-test-into_two_groups_one_with_quorum: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Sync test using synthetic-network - into two groups one with quorum runs-on: ubuntu-20.04 @@ -198,6 +232,7 @@ jobs: test-case: test::sync::into_two_groups_one_with_quorum image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.synthetic-network_sync-tests.yml node-count: 7 # yamllint disable-line rule:line-length @@ -205,7 +240,10 @@ jobs: timeout-minutes: 35 run-e2e-finalization-stall: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Finalization-stall test using synthetic-network runs-on: ubuntu-20.04 steps: @@ -223,56 +261,73 @@ jobs: test-case: test::sync::large_finalization_stall image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet + chain-bootstrapper-image: chain-bootstrapper-image-release compose-file: docker/docker-compose.finalization_stall_with_pruning.yml # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 60 run-major-sync-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run major sync test runs-on: ubuntu-20.04 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-release + path: target/release/ + - name: Run test timeout-minutes: 65 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_major_sync run-force-reorg-test: - needs: [build-production-node-and-runtime] + needs: [build-production-node-and-runtime, build-chain-bootstrapper-production] name: Run force reorgs test runs-on: [self-hosted, Linux, X64, medium] steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Download release artifact + - name: Download node binary from GH artifact uses: actions/download-artifact@v4 with: name: aleph-production-node path: target/release/ + - name: Download chain-bootstrapper from GH artifacts + uses: actions/download-artifact@v4 + with: + name: chain-bootstrapper-release + path: target/release/ + - name: Run test timeout-minutes: 25 run: | ./.github/scripts/test_python_general.sh \ --aleph-node ../target/release/aleph-node \ + --chain-bootstrapper ../target/release/chain-bootstrapper \ --testcase test_force_reorg - run-e2e-no-quorum-without-high-out-latency: - needs: [build-synthetic-node, build-aleph-e2e-client-image] + needs: + - build-synthetic-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper-production name: Run high out-latency for every quorum runs-on: ubuntu-20.04 steps: @@ -286,6 +341,7 @@ jobs: image-path: aleph-release-synthetic-docker node-image: aleph-node:syntheticnet compose-file: docker/docker-compose.synthetic-network.yml + chain-bootstrapper-image: chain-bootstrapper-image-release # yamllint disable-line rule:line-length aleph-e2e-client-image: ${{ needs.build-aleph-e2e-client-image.outputs.aleph-e2e-client-image }} timeout-minutes: 35 diff --git a/.github/workflows/on-main-or-release-branch-commit.yml b/.github/workflows/on-main-or-release-branch-commit.yml index 0b47ab36ad..011aaa661d 100644 --- a/.github/workflows/on-main-or-release-branch-commit.yml +++ b/.github/workflows/on-main-or-release-branch-commit.yml @@ -13,6 +13,25 @@ jobs: uses: ./.github/workflows/_check-vars-and-secrets.yml secrets: inherit + build-chain-bootstrapper: + needs: [check-vars-and-secrets] + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: true + + push-chain-bootstrapper-image-to-ecr: + name: Push chain-bootstrapper docker image to ECR + needs: [build-chain-bootstrapper] + uses: ./.github/workflows/_push-node-image-to-ecr.yml + secrets: inherit + with: + binary-artifact-name: chain-bootstrapper-release + docker-image-name: chain-bootstrapper + binary-name: chain-bootstrapper + docker-file-path: ./bin/chain-bootstrapper/Dockerfile + build-and-push-cliain: needs: [check-vars-and-secrets] name: Build and push cliain @@ -63,8 +82,10 @@ jobs: uses: ./.github/workflows/_push-node-image-to-ecr.yml secrets: inherit with: - node-binary: aleph-production-node + binary-artifact-name: aleph-production-node docker-image-name: aleph-node + binary-name: aleph-node + docker-file-path: ./docker/Dockerfile send-slack-notification-release: name: Send Slack notification about workflow status diff --git a/.github/workflows/on-pull-request-commit.yml b/.github/workflows/on-pull-request-commit.yml index cb91dd95f9..d4e56ca0c6 100644 --- a/.github/workflows/on-pull-request-commit.yml +++ b/.github/workflows/on-pull-request-commit.yml @@ -38,12 +38,19 @@ jobs: name: Check production node and runtime uses: ./.github/workflows/_check-production-node-and-runtime.yml - build-test-node-and-runtime: - name: Build test node and runtime - uses: ./.github/workflows/_build-test-node-and-runtime.yml + build-test-node: + name: Build test node + uses: ./.github/workflows/_build-test-node.yml with: ref: ${{ github.ref }} + build-chain-bootstrapper: + name: Build chain-bootstrapper + uses: ./.github/workflows/_build-chain-bootstrapper.yml + with: + ref: ${{ github.ref }} + production: false + build-aleph-e2e-client-image: name: Build aleph-e2e-client image uses: ./.github/workflows/_build-aleph-e2e-client-image.yml @@ -53,7 +60,10 @@ jobs: run-e2e-finalization-test: name: Run e2e finalization test - needs: [build-test-node-and-runtime, build-aleph-e2e-client-image] + needs: + - build-test-node + - build-aleph-e2e-client-image + - build-chain-bootstrapper runs-on: ubuntu-20.04 steps: - name: Checkout source code diff --git a/BUILD.md b/BUILD.md index 1ab7fd5ec4..82bde88e8b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -106,9 +106,9 @@ After a successful build the binary can be found in `target/release/aleph-node`. ### Local debugging -If you'd like to use Rust debugger with aleph-node binary, built it with `local-debugging` feature: +If you'd like to use Rust debugger with aleph-node binary, built it with `aleph-native-runtime` feature: ``` -cargo check --profile dev -p aleph-node --features local-debugging --locked +cargo check --profile dev -p aleph-node --features aleph-native-runtime --locked ``` [nix]: https://nixos.org/download.html diff --git a/Cargo.lock b/Cargo.lock index fdfe93e995..f0d45d3735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,24 +279,19 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "futures", - "hex", - "hex-literal", "jsonrpsee", - "libp2p", "log", - "pallet-staking", "pallet-transaction-payment-rpc", "parity-scale-codec", "primitives", "sc-basic-authorship", - "sc-chain-spec", "sc-cli", "sc-client-api", + "sc-client-db", "sc-consensus", "sc-consensus-aura", "sc-consensus-slots", "sc-executor", - "sc-keystore", "sc-network", "sc-network-sync", "sc-rpc", @@ -308,7 +303,6 @@ dependencies = [ "serde", "serde_json", "sp-api", - "sp-application-crypto", "sp-arithmetic", "sp-block-builder", "sp-blockchain", @@ -316,7 +310,6 @@ dependencies = [ "sp-consensus-aura", "sp-core", "sp-io", - "sp-keystore", "sp-runtime", "sp-timestamp", "sp-transaction-pool", @@ -1267,6 +1260,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chain-bootstrapper" +version = "0.1.0" +dependencies = [ + "aleph-runtime", + "hex", + "libp2p", + "pallet-staking", + "primitives", + "sc-chain-spec", + "sc-cli", + "sc-client-db", + "sc-keystore", + "sc-service", + "serde", + "serde_json", + "sp-application-crypto", + "sp-core", + "sp-runtime", +] + [[package]] name = "chrono" version = "0.4.38" @@ -3248,12 +3262,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - [[package]] name = "hkdf" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index 6b6d8bdfd0..c2bbfdc731 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "baby-liminal-extension", "bin/node", "bin/runtime", + "bin/chain-bootstrapper", "clique", "finality-aleph", "pallets/aleph", @@ -116,7 +117,8 @@ pallet-proxy = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.gi sc-basic-authorship = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-block-builder = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-chain-spec = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } -sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } +sc-cli = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } +sc-client-db = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } sc-client-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-consensus = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-consensus-aura = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } @@ -128,7 +130,7 @@ sc-network-common = { git = "https://github.com/Cardinal-Cryptography/polkadot-s sc-network-sync = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-rpc = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-rpc-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } -sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } +sc-service = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", default-features = false, branch = "aleph-v1.5.0" } sc-telemetry = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-transaction-pool = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } sc-transaction-pool-api = { git = "https://github.com/Cardinal-Cryptography/polkadot-sdk.git", branch = "aleph-v1.5.0" } diff --git a/bin/chain-bootstrapper/Cargo.toml b/bin/chain-bootstrapper/Cargo.toml new file mode 100644 index 0000000000..ee17515c56 --- /dev/null +++ b/bin/chain-bootstrapper/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "chain-bootstrapper" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +primitives = { workspace = true } +aleph-runtime = { workspace = true } +pallet-staking = { workspace = true } +libp2p = { workspace = true } +hex = { workspace = true } + +sc-cli = { workspace = true, default-features = false } +sc-chain-spec = { workspace = true } +sc-service = { workspace = true, default-features = false } +sc-client-db = { workspace = true, default-features = false } +sc-keystore = { workspace = true } + +sp-application-crypto = { workspace = true } +sp-runtime = { workspace = true } +sp-core = { workspace = true } + +serde_json = { workspace = true } +serde = { workspace = true } + +[features] +default = [] +short_session = [ + "aleph-runtime/short_session", + "primitives/short_session", +] +enable_treasury_proposals = [ + "aleph-runtime/enable_treasury_proposals" +] +try-runtime = [ + "aleph-runtime/try-runtime", +] diff --git a/bin/chain-bootstrapper/Dockerfile b/bin/chain-bootstrapper/Dockerfile new file mode 100644 index 0000000000..92147703a6 --- /dev/null +++ b/bin/chain-bootstrapper/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:jammy + +COPY target/release/chain-bootstrapper /usr/local/bin +RUN chmod +x /usr/local/bin/chain-bootstrapper + +ENTRYPOINT ["chain-bootstrapper"] diff --git a/bin/node/src/chain_spec/builder.rs b/bin/chain-bootstrapper/src/chain_spec/builder.rs similarity index 100% rename from bin/node/src/chain_spec/builder.rs rename to bin/chain-bootstrapper/src/chain_spec/builder.rs diff --git a/bin/node/src/chain_spec/cli.rs b/bin/chain-bootstrapper/src/chain_spec/cli.rs similarity index 100% rename from bin/node/src/chain_spec/cli.rs rename to bin/chain-bootstrapper/src/chain_spec/cli.rs diff --git a/bin/node/src/chain_spec/commands.rs b/bin/chain-bootstrapper/src/chain_spec/commands.rs similarity index 98% rename from bin/node/src/chain_spec/commands.rs rename to bin/chain-bootstrapper/src/chain_spec/commands.rs index 3c8b20f99a..e9cf6a1485 100644 --- a/bin/node/src/chain_spec/commands.rs +++ b/bin/chain-bootstrapper/src/chain_spec/commands.rs @@ -1,5 +1,6 @@ use std::{io::Write, path::PathBuf}; +use primitives::DEFAULT_BACKUP_FOLDER; use sc_cli::{ clap::{self, Parser}, Error, KeystoreParams, @@ -12,7 +13,7 @@ use crate::chain_spec::{ keystore::{ create_abft_backup_dir, create_account_session_keys, create_p2p_key, open_keystore, }, - AlephNodeChainSpec, DEFAULT_BACKUP_FOLDER, + AlephNodeChainSpec, }; /// This command generates session keys and libp2p key for all input accounts. diff --git a/bin/node/src/chain_spec/keystore.rs b/bin/chain-bootstrapper/src/chain_spec/keystore.rs similarity index 98% rename from bin/node/src/chain_spec/keystore.rs rename to bin/chain-bootstrapper/src/chain_spec/keystore.rs index c17caa254e..dd3e54de4c 100644 --- a/bin/node/src/chain_spec/keystore.rs +++ b/bin/chain-bootstrapper/src/chain_spec/keystore.rs @@ -6,11 +6,10 @@ use std::{ use libp2p::identity::ed25519 as libp2p_ed25519; use primitives::{AccountId, AuraId, AuthorityId}; use sc_cli::KeystoreParams; -use sc_keystore::LocalKeystore; +use sc_keystore::{Keystore, LocalKeystore}; use sc_service::{config::KeystoreConfig, BasePath}; use serde::{Deserialize, Serialize}; use sp_core::crypto::key_types; -use sp_keystore::Keystore; #[derive(Clone, Deserialize, Serialize)] pub struct AccountSessionKeys { diff --git a/bin/node/src/chain_spec/mod.rs b/bin/chain-bootstrapper/src/chain_spec/mod.rs similarity index 75% rename from bin/node/src/chain_spec/mod.rs rename to bin/chain-bootstrapper/src/chain_spec/mod.rs index a49e90b0f5..ed90c5dc2b 100644 --- a/bin/node/src/chain_spec/mod.rs +++ b/bin/chain-bootstrapper/src/chain_spec/mod.rs @@ -10,7 +10,6 @@ pub const CHAINTYPE_LOCAL: &str = "local"; pub const CHAINTYPE_LIVE: &str = "live"; pub const DEFAULT_CHAIN_ID: &str = "a0dnet1"; -pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; pub const DEFAULT_SUDO_ACCOUNT_ALICE: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; @@ -20,14 +19,6 @@ use sc_chain_spec::ChainType; use sc_cli::Error; use sp_application_crypto::Ss58Codec; -pub fn mainnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::mainnet_chainspec()) -} - -pub fn testnet_config() -> Result { - AlephNodeChainSpec::from_json_bytes(crate::resources::testnet_chainspec()) -} - fn parse_chaintype(s: &str) -> Result { Ok(match s { CHAINTYPE_DEV => ChainType::Development, diff --git a/bin/chain-bootstrapper/src/main.rs b/bin/chain-bootstrapper/src/main.rs new file mode 100644 index 0000000000..4ba27a10a9 --- /dev/null +++ b/bin/chain-bootstrapper/src/main.rs @@ -0,0 +1,32 @@ +mod chain_spec; + +use sc_cli::clap::{self, Parser, Subcommand as ClapSubcommand}; + +use crate::chain_spec::{BootstrapChainCmd, ConvertChainspecToRawCmd}; + +#[derive(Parser)] +struct Cli { + #[command(subcommand)] + pub subcommand: Option, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, ClapSubcommand)] +pub enum Subcommand { + /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout + BootstrapChain(BootstrapChainCmd), + + /// Takes a chainspec and generates a corresponding raw chainspec + ConvertChainspecToRaw(ConvertChainspecToRawCmd), +} + +fn main() -> sc_cli::Result<()> { + let cli = Cli::parse(); + + match &cli.subcommand { + Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), + Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), + + None => Err("Command was required!".into()), + } +} diff --git a/bin/node/Cargo.toml b/bin/node/Cargo.toml index a416bb9e25..f50bfcb714 100644 --- a/bin/node/Cargo.toml +++ b/bin/node/Cargo.toml @@ -21,48 +21,36 @@ log = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } futures = { workspace = true } -hex = { workspace = true } -hex-literal = { workspace = true } -libp2p = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } -sc-basic-authorship = { workspace = true } -sc-chain-spec = { workspace = true } sc-cli = { workspace = true } +sc-basic-authorship = { workspace = true } sc-client-api = { workspace = true } sc-consensus = { workspace = true } sc-consensus-aura = { workspace = true } sc-consensus-slots = { workspace = true } sc-executor = { workspace = true } -sc-keystore = { workspace = true } sc-network = { workspace = true } sc-network-sync = { workspace = true } sc-service = { workspace = true } +sc-client-db = { workspace = true } sc-telemetry = { workspace = true } sc-transaction-pool = { workspace = true } sc-transaction-pool-api = { workspace = true } -sp-application-crypto = { workspace = true } sp-arithmetic = { workspace = true } sp-block-builder = { workspace = true } sp-consensus = { workspace = true } sp-consensus-aura = { workspace = true } sp-core = { workspace = true } sp-io = { workspace = true } -sp-keystore = { workspace = true } sp-runtime = { workspace = true } sp-timestamp = { workspace = true } sp-transaction-pool = { workspace = true } -pallet-staking = { workspace = true } -try-runtime-cli = { workspace = true, optional = true } - frame-benchmarking-cli = { workspace = true, optional = true } frame-benchmarking = { workspace = true, optional = true } -# this is only neeeded for chainspec generation -aleph-runtime = { workspace = true } - aleph-runtime-interfaces = { workspace = true } finality-aleph = { workspace = true } primitives = { workspace = true } @@ -77,28 +65,47 @@ sp-blockchain = { workspace = true } substrate-frame-rpc-system = { workspace = true } pallet-transaction-payment-rpc = { workspace = true } +aleph-runtime = { workspace = true, optional = true } +try-runtime-cli = { workspace = true, optional = true } + [build-dependencies] substrate-build-script-utils = { workspace = true } [features] -default = [] -short_session = [ - "aleph-runtime/short_session", - "primitives/short_session", -] -try-runtime = [ - "aleph-runtime/try-runtime", - "try-runtime-cli/try-runtime", -] -enable_treasury_proposals = [ - "aleph-runtime/enable_treasury_proposals" +default = ["std", "rocksdb"] +std = [ + "parity-scale-codec/std", + "sp-api/std", + "sp-block-builder/std", + "sp-consensus-aura/std", + "sp-core/std", + "sp-runtime/std", + "sp-transaction-pool/std", + "primitives/std", + "sp-io/std", + "aleph-runtime-interfaces/std", ] runtime-benchmarks = [ + "aleph-runtime-native", "frame-benchmarking", "frame-benchmarking-cli/runtime-benchmarks", - "aleph-runtime/runtime-benchmarks", + "aleph-runtime?/runtime-benchmarks", ] +rocksdb = [ + "sc-cli/rocksdb", + "sc-client-db/rocksdb", + "sc-service/rocksdb", +] +try-runtime = [ + "aleph-runtime-native", + "dep:try-runtime-cli", + "aleph-runtime?/try-runtime", + "try-runtime-cli?/try-runtime", +] +aleph-runtime-native = [ + "dep:aleph-runtime" +] + only_legacy = [ "finality-aleph/only_legacy" ] -local-debugging = [] diff --git a/bin/node/src/cli.rs b/bin/node/src/cli.rs index 565cc9b30a..e2c7b811e8 100644 --- a/bin/node/src/cli.rs +++ b/bin/node/src/cli.rs @@ -3,12 +3,11 @@ use sc_cli::{ PurgeChainCmd, RunCmd, SubstrateCli, }; +pub type AlephNodeChainSpec = sc_service::GenericChainSpec<()>; + use crate::{ aleph_cli::AlephCli, - chain_spec, - chain_spec::{ - commands::BootstrapChainCmd, mainnet_config, testnet_config, ConvertChainspecToRawCmd, - }, + resources::{mainnet_chainspec, testnet_chainspec}, }; #[derive(Debug, Parser)] @@ -24,6 +23,14 @@ pub struct Cli { pub run: RunCmd, } +pub fn mainnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(mainnet_chainspec()) +} + +pub fn testnet_config() -> Result { + AlephNodeChainSpec::from_json_bytes(testnet_chainspec()) +} + impl SubstrateCli for Cli { fn impl_name() -> String { "Aleph Node".into() @@ -58,7 +65,7 @@ impl SubstrateCli for Cli { "mainnet" => mainnet_config(), "testnet" => testnet_config(), - _ => chain_spec::AlephNodeChainSpec::from_json_file(id.into()), + _ => AlephNodeChainSpec::from_json_file(id.into()), }; Ok(Box::new(chainspec?)) } @@ -70,12 +77,6 @@ pub enum Subcommand { #[command(subcommand)] Key(sc_cli::KeySubcommand), - /// Generates keystore (libp2p key and session keys), and generates chainspec to stdout - BootstrapChain(BootstrapChainCmd), - - /// Takes a chainspec and generates a corresponding raw chainspec - ConvertChainspecToRaw(ConvertChainspecToRawCmd), - /// Validate blocks. CheckBlock(sc_cli::CheckBlockCmd), diff --git a/bin/node/src/executor.rs b/bin/node/src/executor.rs index 749e94c41f..71693a3755 100644 --- a/bin/node/src/executor.rs +++ b/bin/node/src/executor.rs @@ -6,7 +6,7 @@ use sc_service::Configuration; #[cfg(not(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" )))] pub mod aleph_executor { @@ -27,7 +27,7 @@ pub mod aleph_executor { #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub mod aleph_executor { diff --git a/bin/node/src/lib.rs b/bin/node/src/lib.rs index bb2731cce3..dc2bf4c39e 100644 --- a/bin/node/src/lib.rs +++ b/bin/node/src/lib.rs @@ -1,6 +1,5 @@ mod aleph_cli; mod aleph_node_rpc; -mod chain_spec; mod cli; mod config; mod executor; @@ -12,7 +11,7 @@ pub use cli::{Cli, Subcommand}; pub use config::Validator as ConfigValidator; #[cfg(any( feature = "runtime-benchmarks", - feature = "local-debugging", + feature = "aleph-native-runtime", feature = "try-runtime" ))] pub use executor::aleph_executor::ExecutorDispatch; diff --git a/bin/node/src/main.rs b/bin/node/src/main.rs index 0a7a7cc54f..7d681c1c9a 100644 --- a/bin/node/src/main.rs +++ b/bin/node/src/main.rs @@ -15,8 +15,6 @@ fn main() -> sc_cli::Result<()> { let config_validation_result = ConfigValidator::process(&mut cli); match &cli.subcommand { - Some(Subcommand::BootstrapChain(cmd)) => cmd.run(), - Some(Subcommand::ConvertChainspecToRaw(cmd)) => cmd.run(), Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::CheckBlock(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/bin/node/src/service.rs b/bin/node/src/service.rs index 978a0cedce..0a3ca0f4ee 100644 --- a/bin/node/src/service.rs +++ b/bin/node/src/service.rs @@ -13,7 +13,8 @@ use finality_aleph::{ }; use log::warn; use primitives::{ - fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, MAX_BLOCK_SIZE, + fake_runtime_api::fake_runtime::RuntimeApi, AlephSessionApi, Block, DEFAULT_BACKUP_FOLDER, + MAX_BLOCK_SIZE, }; use sc_basic_authorship::ProposerFactory; use sc_client_api::{BlockBackend, HeaderBackend}; @@ -29,7 +30,6 @@ use sp_consensus_aura::{sr25519::AuthorityPair as AuraPair, Slot}; use crate::{ aleph_cli::AlephCli, - chain_spec::DEFAULT_BACKUP_FOLDER, executor::aleph_executor, rpc::{create_full as create_full_rpc, FullDeps as RpcFullDeps}, }; diff --git a/local-tests/chainrunner/chain.py b/local-tests/chainrunner/chain.py index e3b66eebec..548dc6a93a 100644 --- a/local-tests/chainrunner/chain.py +++ b/local-tests/chainrunner/chain.py @@ -43,16 +43,16 @@ def new(self, binary, validators, nonvalidators=None): self.nodes = self.validator_nodes + self.nonvalidator_nodes - def bootstrap(self, binary, validators, nonvalidators=None, raw=True, **kwargs): + def bootstrap(self, aleph_node_binary, chain_bootstrapper_binary, validators, nonvalidators=None, raw=True, **kwargs): """Bootstrap the chain. `validators` and `nonvalidators` should be lists of strings with public keys. Flags `--account-ids`, `--base-path` and `--raw` are added automatically. All other flags are taken from kwargs""" - self.new(binary, validators, nonvalidators) + self.new(aleph_node_binary, validators, nonvalidators) nonvalidators = nonvalidators or [] all_accounts = list(validators) + list(nonvalidators) - binary = check_file(binary) - cmd = [binary, 'bootstrap-chain', + binary = check_file(chain_bootstrapper_binary) + cmd = [chain_bootstrapper_binary, 'bootstrap-chain', '--base-path', self.path, '--account-ids', ','.join(all_accounts), '--authorities-account-ids', ','.join(validators)] diff --git a/local-tests/run_nodes.py b/local-tests/run_nodes.py index e801ed2fc7..fff35abd92 100755 --- a/local-tests/run_nodes.py +++ b/local-tests/run_nodes.py @@ -9,19 +9,21 @@ nodes = 4 workdir = '.' -binary = '../target/release/aleph-node' +aleph_node_binary = '../target/release/aleph-node' +chain_bootstrapper_binary = '../target/release/chain-bootstrapper' port = 30334 rpc_port = 9944 phrases = ['//Alice', '//Bob', '//Charlie', '//Dave', '//Ezekiel', '//Fanny', '//George', '//Hugo'] -keys_dict = generate_keys(binary, phrases) +keys_dict = generate_keys(aleph_node_binary, phrases) keys = list(keys_dict.values()) nodes = min(nodes, len(phrases)) chain = Chain(workdir) print(f'Bootstrapping chain for {nodes} nodes') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys[:nodes], chain_type='local') chain.set_flags('validator', diff --git a/local-tests/test_force_reorg.py b/local-tests/test_force_reorg.py index eb9ee1cb50..90da775088 100755 --- a/local-tests/test_force_reorg.py +++ b/local-tests/test_force_reorg.py @@ -12,14 +12,17 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary (important DON'T use short-session feature): -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) + phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_major_sync.py b/local-tests/test_major_sync.py index 561d665d49..f9d1c29a8a 100755 --- a/local-tests/test_major_sync.py +++ b/local-tests/test_major_sync.py @@ -12,15 +12,19 @@ workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) logging.info(f"Workdir is {workdir}") -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) -logging.info(f"aleph-node binary is {binary}") +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) +logging.info(f"aleph-node binary is {aleph_node_binary}") +logging.info(f"chain_bootstrapper binary is {chain_bootstrapper_binary}") + phrases = [f'//{i}' for i in range(5)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/local-tests/test_recover_after_abft_update.py b/local-tests/test_recover_after_abft_update.py index 8f40586160..8b8926204d 100755 --- a/local-tests/test_recover_after_abft_update.py +++ b/local-tests/test_recover_after_abft_update.py @@ -17,14 +17,16 @@ # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary. -binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +aleph_node_binary = abspath(os.getenv('ALEPH_NODE_BINARY', join(workdir, 'aleph-node'))) +chain_bootstrapper_binary = abspath(os.getenv('CHAIN_BOOTSTRAPPER', join(workdir, 'chain-bootstrapper'))) phrases = [f'//{i}' for i in range(8)] -keys = generate_keys(binary, phrases) +keys = generate_keys(aleph_node_binary, phrases) chain = Chain(workdir) logging.info('Bootstrapping the chain with binary') -chain.bootstrap(binary, +chain.bootstrap(aleph_node_binary, + chain_bootstrapper_binary, keys.values(), sudo_account_id=keys[phrases[0]], chain_type='local') diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index c102864fb6..ee9b60a14d 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -159,6 +159,9 @@ pub const LENIENT_THRESHOLD: Perquintill = Perquintill::from_percent(90); /// Number of non-finalized blocks that halts block production pub const DEFAULT_MAX_NON_FINALIZED_BLOCKS: u32 = 20; +/// A relative folder where to store ABFT backups +pub const DEFAULT_BACKUP_FOLDER: &str = "backup-stash"; + /// Hold set of validators that produce blocks and set of validators that participate in finality /// during session. #[derive(Decode, Encode, TypeInfo, Debug, Clone, PartialEq, Eq)] diff --git a/scripts/run_nodes.sh b/scripts/run_nodes.sh index bb38ce3107..cb98f9bf7b 100755 --- a/scripts/run_nodes.sh +++ b/scripts/run_nodes.sh @@ -39,7 +39,8 @@ set -euo pipefail # ------------------------ constants -------------------------------------- -export ALEPH_NODE="target/release/aleph-node" +ALEPH_NODE="target/release/aleph-node" +CHAINSPEC_GENERATOR="target/release/chain-bootstrapper" NODE_P2P_PORT_RANGE_START=30333 NODE_VALIDATOR_PORT_RANGE_START=30343 NODE_RPC_PORT_RANGE_START=9944 @@ -193,8 +194,11 @@ function run_node() { # ------------------------- input checks ---------------------------------- +if [[ "${VALIDATORS}" -lt 1 ]]; then + error "Number of validators should be at least 1!" +fi if [[ "${VALIDATORS}" -lt 4 ]]; then - error "Number of validators should be at least 4!" + warning "AlephBFT is only supported for more than 4 nodes." fi if [[ "${RPC_NODES}" -lt 1 ]]; then error "Number of RPC nodes should be at least 1!" @@ -227,10 +231,13 @@ if ! killall -9 aleph-node 2> /dev/null; then fi if [[ -z "${DONT_BUILD_ALEPH_NODE}" ]]; then - info "Building testing aleph-node binary (short session)." - cargo build --release -p aleph-node --features "short_session enable_treasury_proposals" -elif [[ ! -x "${ALEPH_NODE}" ]]; then - error "${ALEPH_NODE} does not exist or it's not an executable file!" + info "Building testing aleph-node binary (short session) and chain-bootstrapper binary." + cargo build --release -p aleph-node + if [[ -z "${DONT_BOOTSTRAP}" ]]; then + cargo build --release -p chain-bootstrapper --features "short_session enable_treasury_proposals" + fi +elif [[ ! -x "${ALEPH_NODE}" || ! -x "${CHAINSPEC_GENERATOR}" ]]; then + error "${ALEPH_NODE} or ${CHAINSPEC_GENERATOR} does not exist or it's not an executable file!" fi NUMBER_OF_NODES_TO_BOOTSTRAP=$(( VALIDATORS + RPC_NODES )) @@ -263,7 +270,7 @@ if [[ -z "${DONT_BOOTSTRAP}" ]]; then validator_ids_string="${validator_ids_string//${IFS:0:1}/,}" info "Populating keystore for all accounts with session keys and libp2p key, and generating chainspec" - "${ALEPH_NODE}" bootstrap-chain \ + "${CHAINSPEC_GENERATOR}" bootstrap-chain \ --raw \ --base-path "${BASE_PATH}" \ --account-ids "${all_account_ids_string}" \