diff --git a/.dockerignore b/.dockerignore index 9d62f3c5c13..b26a7d4f413 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,3 +21,4 @@ !zebra-* !zebrad !docker/entrypoint.sh +!docker/default_zebra_config.toml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 68d83f0fd1e..e45be336730 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -223,7 +223,6 @@ docker run --rm -e TEST_LWD_INTEGRATION=1 zebra-tests #### Test Categories - Full suite (`RUN_ALL_TESTS`) -- Experimental features (`RUN_ALL_EXPERIMENTAL_TESTS`) - Integration tests (`TEST_LWD_INTEGRATION`) - Network sync (`TEST_ZEBRA_EMPTY_SYNC`, `TEST_UPDATE_SYNC`) - State management (`TEST_DISK_REBUILD`) diff --git a/.github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/cd-deploy-nodes-gcp.yml index 315a7dc4464..0f015429d62 100644 --- a/.github/workflows/cd-deploy-nodes-gcp.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -1,6 +1,6 @@ # Google Cloud node deployments and tests that run when Rust code or dependencies are modified, # but only on PRs from the ZcashFoundation/zebra repository. -# (External PRs are tested/deployed by GitHub's Merge Queue.) +# (External PRs are tested/deployed by GitHub's Merge Queue.) # # 1. `versioning`: Extracts the major version from the release semver. Useful for segregating instances based on major versions. # 2. `build`: Builds a Docker image named `zebrad` with the necessary tags derived from Git. @@ -28,13 +28,13 @@ concurrency: on: merge_group: - types: [ checks_requested ] + types: [checks_requested] workflow_dispatch: inputs: network: default: Mainnet - description: 'Network to deploy: Mainnet or Testnet' + description: "Network to deploy: Mainnet or Testnet" required: true type: choice options: @@ -42,7 +42,7 @@ on: - Testnet cached_disk_type: default: tip - description: 'Type of cached disk to use' + description: "Type of cached disk to use" required: true type: choice options: @@ -50,63 +50,63 @@ on: - checkpoint prefer_main_cached_state: default: false - description: 'Prefer cached state from the main branch' + description: "Prefer cached state from the main branch" required: false type: boolean need_cached_disk: default: true - description: 'Use a cached state disk' + description: "Use a cached state disk" required: false type: boolean no_cache: - description: 'Disable the Docker cache for this build' + description: "Disable the Docker cache for this build" required: false type: boolean default: false log_file: - default: '' - description: 'Log to a file path rather than standard output' + default: "" + description: "Log to a file path rather than standard output" push: - # Skip main branch updates where Rust code and dependencies aren't modified. - branches: - - main - paths: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/cd-deploy-nodes-gcp.yml' - - '.github/workflows/sub-build-docker-image.yml' + # Skip main branch updates where Rust code and dependencies aren't modified. + branches: + - main + paths: + # code and tests + - "**/*.rs" + # hard-coded checkpoints and proptest regressions + - "**/*.txt" + # dependencies + - "**/Cargo.toml" + - "**/Cargo.lock" + # configuration files + - ".cargo/config.toml" + - "**/clippy.toml" + # workflow definitions + - "docker/**" + - ".dockerignore" + - ".github/workflows/cd-deploy-nodes-gcp.yml" + - ".github/workflows/sub-build-docker-image.yml" # Only runs the Docker image tests, doesn't deploy any instances pull_request: # Skip PRs where Rust code and dependencies aren't modified. paths: # code and tests - - '**/*.rs' + - "**/*.rs" # hard-coded checkpoints and proptest regressions - - '**/*.txt' + - "**/*.txt" # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' + - "**/Cargo.toml" + - "**/Cargo.lock" # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' + - ".cargo/config.toml" + - "**/clippy.toml" # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/cd-deploy-nodes-gcp.yml' - - '.github/workflows/sub-build-docker-image.yml' + - "docker/**" + - ".dockerignore" + - ".github/workflows/cd-deploy-nodes-gcp.yml" + - ".github/workflows/sub-build-docker-image.yml" release: types: @@ -160,6 +160,17 @@ jobs: disk_suffix: ${{ inputs.cached_disk_type || 'tip' }} prefer_main_cached_state: ${{ inputs.prefer_main_cached_state || (github.event_name == 'push' && github.ref_name == 'main' && true) || false }} + # Test that Zebra works using $ZEBRA_CONF_PATH config + test-zebra-conf-path: + name: Test CD custom Docker config file + needs: build + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: "custom-conf" + docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' + grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v2.1.0.toml"' + # Each time this workflow is executed, a build will be triggered to create a new image # with the corresponding tags using information from Git # @@ -174,6 +185,7 @@ jobs: image_name: zebrad no_cache: ${{ inputs.no_cache || false }} rust_log: info + features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES) }} # This step needs access to Docker Hub secrets to run successfully secrets: inherit @@ -183,11 +195,9 @@ jobs: needs: build uses: ./.github/workflows/sub-test-zebra-config.yml with: - test_id: 'default-conf' + test_id: "default-conf" docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"' - test_variables: '-e NETWORK' - network: 'Mainnet' # Test reconfiguring the docker image for testnet. test-configuration-file-testnet: @@ -196,23 +206,10 @@ jobs: # Make sure Zebra can sync the genesis block on testnet uses: ./.github/workflows/sub-test-zebra-config.yml with: - test_id: 'testnet-conf' + test_id: "testnet-conf" docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' - test_variables: '-e NETWORK' - network: 'Testnet' - - # Test that Zebra works using $ZEBRA_CONF_PATH config - test-zebra-conf-path: - name: Test CD custom Docker config file - needs: build - uses: ./.github/workflows/sub-test-zebra-config.yml - with: - test_id: 'custom-conf' - docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v1.0.0-rc.2.toml"' - test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' - network: ${{ inputs.network || vars.ZCASH_NETWORK }} + test_variables: "-e NETWORK=Testnet" # Deploy Managed Instance Groups (MiGs) for Mainnet and Testnet, # with one node in the configured GCP region. @@ -234,15 +231,22 @@ jobs: matrix: network: [Mainnet, Testnet] name: Deploy ${{ matrix.network }} nodes - needs: [ build, versioning, test-configuration-file, test-zebra-conf-path, get-disk-name ] + needs: + [ + build, + versioning, + test-configuration-file, + test-zebra-conf-path, + get-disk-name, + ] runs-on: ubuntu-latest timeout-minutes: 60 env: CACHED_DISK_NAME: ${{ needs.get-disk-name.outputs.cached_disk_name }} environment: ${{ github.event_name == 'release' && 'prod' || 'dev' }} permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" if: ${{ !cancelled() && !failure() && ((github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'release') }} steps: @@ -271,8 +275,8 @@ jobs: id: auth uses: google-github-actions/auth@v2.1.8 with: - workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + workload_identity_provider: "${{ vars.GCP_WIF }}" + service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}" - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2.1.4 @@ -301,11 +305,11 @@ jobs: --image-family=cos-stable \ --network-interface=subnet=${{ vars.GCP_SUBNETWORK }} \ --create-disk="${DISK_PARAMS}" \ - --container-mount-disk=mount-path='/var/cache/zebrad-cache',name=${DISK_NAME},mode=rw \ + --container-mount-disk=mount-path='/home/zebra/.cache/zebra',name=${DISK_NAME},mode=rw \ --container-stdin \ --container-tty \ --container-image ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \ - --container-env "NETWORK=${{ matrix.network }},LOG_FILE=${{ vars.CD_LOG_FILE }},LOG_COLOR=false,SENTRY_DSN=${{ vars.SENTRY_DSN }}" \ + --container-env "NETWORK=${{ matrix.network }},LOG_FILE=${{ vars.CD_LOG_FILE }},SENTRY_DSN=${{ vars.SENTRY_DSN }}" \ --service-account ${{ vars.GCP_DEPLOYMENTS_SA }} \ --scopes cloud-platform \ --metadata google-logging-enabled=true,google-logging-use-fluentbit=true,google-monitoring-enabled=true \ @@ -349,14 +353,14 @@ jobs: # Note: this instances are not automatically replaced or deleted deploy-instance: name: Deploy single ${{ inputs.network }} instance - needs: [ build, test-configuration-file, test-zebra-conf-path, get-disk-name ] + needs: [build, test-configuration-file, test-zebra-conf-path, get-disk-name] runs-on: ubuntu-latest timeout-minutes: 30 env: CACHED_DISK_NAME: ${{ needs.get-disk-name.outputs.cached_disk_name }} permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" # Run even if we don't need a cached disk, but only when triggered by a workflow_dispatch if: ${{ !failure() && github.event_name == 'workflow_dispatch' }} @@ -386,8 +390,8 @@ jobs: id: auth uses: google-github-actions/auth@v2.1.8 with: - workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + workload_identity_provider: "${{ vars.GCP_WIF }}" + service_account: "${{ vars.GCP_DEPLOYMENTS_SA }}" - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v2.1.4 @@ -413,11 +417,11 @@ jobs: --image-family=cos-stable \ --network-interface=subnet=${{ vars.GCP_SUBNETWORK }} \ --create-disk="${DISK_PARAMS}" \ - --container-mount-disk=mount-path='/var/cache/zebrad-cache',name=${DISK_NAME},mode=rw \ + --container-mount-disk=mount-path='/home/zebra/.cache/zebra',name=${DISK_NAME},mode=rw \ --container-stdin \ --container-tty \ --container-image ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \ - --container-env "NETWORK=${{ inputs.network }},LOG_FILE=${{ inputs.log_file }},LOG_COLOR=false,SENTRY_DSN=${{ vars.SENTRY_DSN }}" \ + --container-env "NETWORK=${{ inputs.network }},LOG_FILE=${{ inputs.log_file }},SENTRY_DSN=${{ vars.SENTRY_DSN }}" \ --service-account ${{ vars.GCP_DEPLOYMENTS_SA }} \ --scopes cloud-platform \ --metadata google-logging-enabled=true,google-monitoring-enabled=true \ @@ -428,7 +432,7 @@ jobs: failure-issue: name: Open or update issues for release failures # When a new job is added to this workflow, add it to this list. - needs: [ versioning, build, deploy-nodes, deploy-instance ] + needs: [versioning, build, deploy-nodes, deploy-instance] # Only open tickets for failed or cancelled jobs that are not coming from PRs. # (PR statuses are already reported in the PR jobs list, and checked by GitHub's Merge Queue.) if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 517ba4151f2..ffb886b4512 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -133,6 +133,7 @@ jobs: rust_backtrace: full rust_lib_backtrace: full rust_log: info + features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES) }} # This step needs access to Docker Hub secrets to run successfully secrets: inherit diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml index c5142babe31..1ec1b88c0d2 100644 --- a/.github/workflows/sub-build-docker-image.yml +++ b/.github/workflows/sub-build-docker-image.yml @@ -32,14 +32,9 @@ on: rust_log: required: false type: string - # defaults to: vars.RUST_PROD_FEATURES features: required: false type: string - # defaults to: vars.RUST_TEST_FEATURES (and entrypoint.sh adds vars.RUST_PROD_FEATURES) - test_features: - required: false - type: string latest_tag: required: false type: boolean @@ -48,20 +43,18 @@ on: required: false type: string no_cache: - description: 'Disable the Docker cache for this build' + description: "Disable the Docker cache for this build" required: false type: boolean default: false outputs: image_digest: - description: 'The image digest to be used on a caller workflow' + description: "The image digest to be used on a caller workflow" value: ${{ jobs.build.outputs.image_digest }} - env: - FEATURES: ${{ inputs.features || vars.RUST_PROD_FEATURES }} - TEST_FEATURES: ${{ inputs.test_features || vars.RUST_TEST_FEATURES }} + FEATURES: ${{ inputs.features }} RUST_LOG: ${{ inputs.rust_log || vars.RUST_LOG }} CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} @@ -75,8 +68,8 @@ jobs: image_digest: ${{ steps.docker_build.outputs.digest }} image_name: ${{ fromJSON(steps.docker_build.outputs.metadata)['image.name'] }} permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" pull-requests: write # for `docker-scout` to be able to write the comment env: DOCKER_BUILD_SUMMARY: ${{ vars.DOCKER_BUILD_SUMMARY }} @@ -129,9 +122,9 @@ jobs: id: auth uses: google-github-actions/auth@v2.1.8 with: - workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_ARTIFACTS_SA }}' - token_format: 'access_token' + workload_identity_provider: "${{ vars.GCP_WIF }}" + service_account: "${{ vars.GCP_ARTIFACTS_SA }}" + token_format: "access_token" # Some builds might take over an hour, and Google's default lifetime duration for # an access token is 1 hour (3600s). We increase this to 3 hours (10800s) # as some builds take over an hour. @@ -173,7 +166,6 @@ jobs: SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} RUST_LOG=${{ env.RUST_LOG }} FEATURES=${{ env.FEATURES }} - TEST_FEATURES=${{ env.TEST_FEATURES }} push: true # It's recommended to build images with max-level provenance attestations # https://docs.docker.com/build/ci/github-actions/attestations/ diff --git a/.github/workflows/sub-ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml index d4e4bd506d4..438b32ac235 100644 --- a/.github/workflows/sub-ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -32,7 +32,7 @@ on: #! The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and #! `ci-integration-tests-gcp.patch-external.yml` must be kept in sync. #! -#! The test variables ZEBRA_CACHED_STATE_DIR and LIGHTWALLETD_DATA_DIR used in some steps are set in the +#! The test variables ZEBRA_CACHE_DIR and LWD_CACHE_DIR used in some steps are set in the #! `sub-deploy-integration-tests-gcp.yml` workflow file as inputs. If modified in this file, they must #! also be updated in the `sub-deploy-integration-tests-gcp.yml` file. jobs: @@ -79,7 +79,7 @@ jobs: app_name: zebrad test_id: sync-to-checkpoint test_description: Test sync up to mandatory checkpoint - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1" needs_zebra_state: false saves_to_disk: true force_save_to_disk: ${{ inputs.force_save_to_disk || false }} @@ -108,7 +108,7 @@ jobs: app_name: zebrad test_id: sync-past-checkpoint test_description: Test full validation sync from a cached state - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1" needs_zebra_state: true saves_to_disk: false disk_suffix: checkpoint @@ -138,7 +138,7 @@ jobs: test_description: Test a full sync up to the tip # The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored. # TODO: update the test to use {{ input.network }} instead? - test_variables: "-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0" # This test runs for longer than 6 hours, so it needs multiple jobs is_long_test: true needs_zebra_state: false @@ -178,7 +178,7 @@ jobs: app_name: zebrad test_id: update-to-tip test_description: Test syncing to tip with a Zebra tip state - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1" needs_zebra_state: true # update the disk on every PR, to increase CI speed saves_to_disk: true @@ -209,7 +209,7 @@ jobs: test_id: checkpoints-mainnet test_description: Generate Zebra checkpoints on mainnet # TODO: update the test to use {{ input.network }} instead? - test_variables: "-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1" needs_zebra_state: true # test-update-sync updates the disk on every PR, so we don't need to do it here saves_to_disk: false @@ -241,7 +241,7 @@ jobs: test_id: full-sync-testnet test_description: Test a full sync up to the tip on testnet # The value of FULL_SYNC_TESTNET_TIMEOUT_MINUTES is currently ignored. - test_variables: "-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0" network: "Testnet" # A full testnet sync could take 2-10 hours in April 2023. # The time varies a lot due to the small number of nodes. @@ -285,7 +285,7 @@ jobs: app_name: zebrad test_id: checkpoints-testnet test_description: Generate Zebra checkpoints on testnet - test_variables: "-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1" network: "Testnet" needs_zebra_state: true # update the disk on every PR, to increase CI speed @@ -316,7 +316,7 @@ jobs: app_name: lightwalletd test_id: lwd-full-sync test_description: Test lightwalletd full sync - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1" # This test runs for longer than 6 hours, so it needs multiple jobs is_long_test: true needs_zebra_state: true @@ -351,7 +351,7 @@ jobs: app_name: lightwalletd test_id: lwd-update-sync test_description: Test lightwalletd update sync with both states - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1" needs_zebra_state: true needs_lwd_state: true saves_to_disk: true @@ -379,7 +379,7 @@ jobs: app_name: lightwalletd test_id: fully-synced-rpc test_description: Test lightwalletd RPC with a Zebra tip state - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1" needs_zebra_state: true saves_to_disk: false secrets: inherit @@ -401,7 +401,7 @@ jobs: app_name: lightwalletd test_id: lwd-send-transactions test_description: Test sending transactions via lightwalletd - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1" needs_zebra_state: true needs_lwd_state: true saves_to_disk: false @@ -424,7 +424,7 @@ jobs: app_name: lightwalletd test_id: lwd-grpc-wallet test_description: Test gRPC calls via lightwalletd - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1" needs_zebra_state: true needs_lwd_state: true saves_to_disk: false @@ -451,7 +451,7 @@ jobs: app_name: zebrad test_id: get-block-template test_description: Test getblocktemplate RPC method via Zebra's rpc server - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1" needs_zebra_state: true needs_lwd_state: false saves_to_disk: false @@ -474,7 +474,7 @@ jobs: app_name: zebrad test_id: submit-block test_description: Test submitting blocks via Zebra's rpc server - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1" needs_zebra_state: true needs_lwd_state: false saves_to_disk: false @@ -488,16 +488,16 @@ jobs: # # If the state version has changed, waits for the new cached states to be created. # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - scan-task-commands-test: - name: scan task commands + test-scanner: + name: Scanner tests needs: [test-full-sync, get-available-disks] uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebra-scan - test_id: scan-task-commands - test_description: Test that the scan task registers keys, deletes keys, and subscribes to results for keys while running. - test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCAN_TASK_COMMANDS=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache" + test_id: scanner-tests + test_description: Tests the scanner. + test_variables: "-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SCANNER=1" needs_zebra_state: true needs_lwd_state: false saves_to_disk: false diff --git a/.github/workflows/sub-ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml index 0847806f610..ab4d9a97f1d 100644 --- a/.github/workflows/sub-ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -20,7 +20,7 @@ on: type: string network: type: string - default: 'Mainnet' + default: "Mainnet" no_cache: type: boolean default: false @@ -32,14 +32,7 @@ env: COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }} CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} -#! IMPORTANT -#! -#! The job names in `ci-unit-tests-docker.yml`, `ci-unit-tests-docker.patch.yml` and -#! `ci-unit-tests-docker.patch-external.yml` must be kept in sync. jobs: - # Run all the zebra tests, including tests that are ignored by default. - # - # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. test-all: name: Test all timeout-minutes: 180 @@ -52,25 +45,19 @@ jobs: with: short-length: 7 - # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails. - # - # If some tests hang, add "-- --nocapture" for just that test, or for all the tests. - # - - name: Run zebrad tests - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e RUN_ALL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - - # Run unit, basic acceptance tests, and ignored tests with experimental features. + # Run unit, basic acceptance tests, and ignored tests, only showing + # command output if the test fails. # - - name: Run zebrad tests with experimental features - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + # If some tests hang, add "-- --nocapture" for just that test, or for all + # the tests. + - name: Run all tests run: | docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e RUN_ALL_EXPERIMENTAL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run -t \ + -e RUN_ALL_TESTS=1 \ + -e FEATURES="journald prometheus filter-reload" \ + -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \ + ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Run state tests with fake activation heights. # @@ -98,7 +85,10 @@ jobs: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run -t \ + -e TEST_FAKE_ACTIVATION_HEIGHTS=1 \ + -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \ + ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. test-empty-sync: @@ -118,7 +108,7 @@ jobs: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e TEST_ZEBRA_EMPTY_SYNC=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e TEST_ZEBRA_EMPTY_SYNC=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test launching lightwalletd with an empty lightwalletd and Zebra state. test-lightwalletd-integration: @@ -138,42 +128,37 @@ jobs: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD=1 -e TEST_LWD_INTEGRATION=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e ZEBRA_TEST_LIGHTWALLETD=1 -e TEST_LWD_INTEGRATION=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test that Zebra works using the default config with the latest Zebra version. test-configuration-file: name: Test CI default Docker config file uses: ./.github/workflows/sub-test-zebra-config.yml with: - test_id: 'default-conf' + test_id: "default-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"' - test_variables: '-e NETWORK' - network: 'Mainnet' - # Test reconfiguring the docker image for tesnet. + # Test reconfiguring Zebra for Testnet in Docker. test-configuration-file-testnet: - name: Test CI testnet Docker config file + name: Test enabling Testnet in Docker # Make sure Zebra can sync the genesis block on testnet uses: ./.github/workflows/sub-test-zebra-config.yml with: - test_id: 'testnet-conf' + test_id: "testnet-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' - # TODO: improve the entrypoint to avoid using `ENTRYPOINT_FEATURES=""` - test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="/etc/zebrad/zebrad.toml" -e ENTRYPOINT_FEATURES=""' - network: 'Testnet' + test_variables: "-e NETWORK=Testnet" # Test that Zebra works using $ZEBRA_CONF_PATH config test-zebra-conf-path: name: Test CI custom Docker config file uses: ./.github/workflows/sub-test-zebra-config.yml with: - test_id: 'custom-conf' + test_id: "custom-conf" docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v1.0.0-rc.2.toml"' - test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' - network: ${{ inputs.network || vars.ZCASH_NETWORK }} + grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v2.1.0.toml"' + test_variables: '-e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v2.1.0.toml"' failure-issue: name: Open or update issues for main branch failures @@ -181,7 +166,15 @@ jobs: # # This list is for reliable tests that are run on the `main` branch. # Testnet jobs are not in this list, because we expect testnet to fail occasionally. - needs: [ test-all, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path ] + needs: + [ + test-all, + test-fake-activation-heights, + test-empty-sync, + test-lightwalletd-integration, + test-configuration-file, + test-zebra-conf-path, + ] # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. # (PR statuses are already reported in the PR jobs list, and checked by GitHub's Merge Queue.) # TODO: if a job times out, we want to create a ticket. Does failure() do that? Or do we need cancelled()? @@ -202,7 +195,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'A-release') runs-on: ubuntu-latest steps: - - name: Run check_no_git_refs_in_cargo_lock - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - docker run --tty -e NETWORK -e RUN_CHECK_NO_GIT_REFS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + - name: Run check_no_git_refs_in_cargo_lock + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e RUN_CHECK_NO_GIT_REFS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} diff --git a/.github/workflows/sub-deploy-integration-tests-gcp.yml b/.github/workflows/sub-deploy-integration-tests-gcp.yml index 1a8854febd0..b054814355f 100644 --- a/.github/workflows/sub-deploy-integration-tests-gcp.yml +++ b/.github/workflows/sub-deploy-integration-tests-gcp.yml @@ -38,12 +38,12 @@ on: zebra_state_dir: required: false type: string - default: '/var/cache/zebrad-cache' + default: '/home/zebra/.cache/zebra' description: 'Zebra cached state directory and input image prefix to search in GCP' lwd_state_dir: required: false type: string - default: '/var/cache/lwd-cache' + default: '/home/zebra/.cache/lwd' description: 'Lightwalletd cached state directory and input image prefix to search in GCP' disk_prefix: required: false @@ -250,7 +250,7 @@ jobs: # `ci-unit-tests-docker.yml` to be able to run this tests. # # Although we're mounting the disk root to both directories, Zebra and Lightwalletd, tests - # will only respect the values from $ZEBRA_CACHED_STATE_DIR and $LIGHTWALLETD_DATA_DIR, + # will only respect the values from $ZEBRA_CACHE_DIR and $LWD_CACHE_DIR, # the inputs like ${{ inputs.zebra_state_dir }} and ${{ inputs.lwd_state_dir }} # are only used to match those variables paths. - name: Launch ${{ inputs.test_id }} test diff --git a/.github/workflows/sub-test-zebra-config.yml b/.github/workflows/sub-test-zebra-config.yml index 69990bdb3e1..d72f35bf4e4 100644 --- a/.github/workflows/sub-test-zebra-config.yml +++ b/.github/workflows/sub-test-zebra-config.yml @@ -23,14 +23,9 @@ on: type: string description: 'Docker image to test' test_variables: - required: true - type: string - description: 'Environment variables used to select and configure the test' - network: required: false type: string - default: Mainnet - description: 'Zcash network to test against' + description: 'Environment variables used to select and configure the test' jobs: test-docker-config: @@ -87,5 +82,3 @@ jobs: # Handle other potential errors here echo "An error occurred while processing the logs."; exit 1; - env: - NETWORK: '${{ inputs.network }}' diff --git a/book/src/user/docker.md b/book/src/user/docker.md index 1349ee6b322..7709bdca4e7 100644 --- a/book/src/user/docker.md +++ b/book/src/user/docker.md @@ -27,7 +27,7 @@ docker volume create zebrad-cache docker run -d --platform linux/amd64 \ --restart unless-stopped \ --env-file .env \ - --mount type=volume,source=zebrad-cache,target=/var/cache/zebrad-cache \ + --mount type=volume,source=zebrad-cache,target=/home/zebra/.cache/zebra \ -p 8233:8233 \ --memory 16G \ --cpus 4 \ @@ -78,7 +78,7 @@ Based on our actual `entrypoint.sh` script, the following configuration file wil network = "Mainnet" listen_addr = "0.0.0.0" [state] -cache_dir = "/var/cache/zebrad-cache" +cache_dir = "/home/zebra/.cache/zebra" [metrics] endpoint_addr = "127.0.0.1:9999" ``` @@ -129,7 +129,6 @@ This approach ensures you can run the same tests locally that are run in CI, pro #### Configuration - `FEATURES`: Specifies the features to build `zebrad` with. Example: `"default-release-binaries getblocktemplate-rpcs"` -- `TEST_FEATURES`: Specifies the features for tests. Example: `"lightwalletd-grpc-tests zebra-checkpoints"` #### Logging @@ -140,9 +139,7 @@ This approach ensures you can run the same tests locally that are run in CI, pro #### Tests -- `TEST_FEATURES`: Specifies the features for tests. Example: `"lightwalletd-grpc-tests zebra-checkpoints"` - `ZEBRA_SKIP_IPV6_TESTS`: Skips IPv6 tests. Example: `1` -- `ENTRYPOINT_FEATURES`: Overrides the specific features used to run tests in `entrypoint.sh`. Example: `"default-release-binaries lightwalletd-grpc-tests"` #### CI/CD @@ -154,14 +151,11 @@ This approach ensures you can run the same tests locally that are run in CI, pro #### Zebra Configuration -- `ZEBRA_CHECKPOINT_SYNC`: Enables or disables checkpoint sync. Example: `true` -- `ZEBRA_LISTEN_ADDR`: Address for Zebra to listen on. Example: `"0.0.0.0"` -- `ZEBRA_CACHED_STATE_DIR`: Directory for cached state. Example: `"/var/cache/zebrad-cache"` +- `ZEBRA_CACHE_DIR`: Directory for cached state. Example: `"/home/zebra/.cache/zebra"` #### Mining Configuration - `RPC_LISTEN_ADDR`: Address for RPC to listen on. Example: `"0.0.0.0"` -- `RPC_PORT`: Port for RPC. Example: `8232` - `MINER_ADDRESS`: Address for the miner. Example: `"t1XhG6pT9xRqRQn3BHP7heUou1RuYrbcrCc"` #### Other Configuration diff --git a/book/src/user/mining-docker.md b/book/src/user/mining-docker.md index 002848c0ca3..96f47918763 100644 --- a/book/src/user/mining-docker.md +++ b/book/src/user/mining-docker.md @@ -20,9 +20,7 @@ meaning it is a Mainnet P2PKH address. Please remember to set your own address for the rewards. The port we mapped between the container and the host with the `-p` flag in the -example above is Zebra's default Mainnet RPC port. If you want to use a -different one, you can specify it in the `RPC_PORT` environment variable, -similarly to `MINER_ADDRESS`, and then map it with the Docker's `-p` flag. +example above is Zebra's default Mainnet RPC port. Instead of listing the environment variables on the command line, you can use Docker's `--env-file` flag to specify a file containing the variables. You diff --git a/docker/Dockerfile b/docker/Dockerfile index c441ce44e22..320ff15aec4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,25 +3,18 @@ # If you want to include a file in the Docker image, add it to .dockerignore. # -# We are using 4 stages: -# - deps: install build dependencies and sets the needed variables +# We are using 4 (TODO: 5) stages: +# - deps: installs build dependencies and sets default values # - tests: builds tests binaries # - release: builds release binaries # - runtime: runs the release binaries +# - TODO: Add a `monitoring` stage # # We first set default values for build arguments used across the stages. # Each stage must define the build arguments (ARGs) it uses. -# -# Build zebrad with these features -# -# Keep these argument defaults in sync with GitHub vars.RUST_PROD_FEATURES and vars.RUST_TEST_FEATURES -# https://github.com/ZcashFoundation/zebra/settings/variables/actions -ARG FEATURES="default-release-binaries" -ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" -ARG EXPERIMENTAL_FEATURES="" - -ARG APP_HOME="/opt/zebrad" -ARG RUST_VERSION=1.82.0 + +ARG RUST_VERSION=1.84.0 + # In this stage we download all system requirements to build the project # # It also captures all the build arguments to be used as environment variables. @@ -29,25 +22,14 @@ ARG RUST_VERSION=1.82.0 FROM rust:${RUST_VERSION}-bookworm AS deps SHELL ["/bin/bash", "-xo", "pipefail", "-c"] -# Set the default path for the zebrad binary -ARG APP_HOME -ENV APP_HOME=${APP_HOME} -WORKDIR ${APP_HOME} - -# Install zebra build deps and Dockerfile deps +# Install zebra build deps RUN apt-get -qq update && \ apt-get -qq install -y --no-install-recommends \ - llvm \ libclang-dev \ - clang \ - ca-certificates \ protobuf-compiler \ - rocksdb-tools \ && rm -rf /var/lib/apt/lists/* /tmp/* # Build arguments and variables set for tracelog levels and debug information -# -# We set defaults to all variables. ARG RUST_LOG ENV RUST_LOG=${RUST_LOG:-info} @@ -65,38 +47,42 @@ ARG SHORT_SHA # https://github.com/ZcashFoundation/zebra/blob/9ebd56092bcdfc1a09062e15a0574c94af37f389/zebrad/src/application.rs#L179-L182 ENV SHORT_SHA=${SHORT_SHA:-} -ENV CARGO_HOME="${APP_HOME}/.cargo/" - -# Copy the entrypoint script to be used on both images -COPY ./docker/entrypoint.sh /etc/zebrad/entrypoint.sh - -# In this stage we build tests (without running then) +# This stage builds tests without running them. # # We also download needed dependencies for tests to work, from other images. # An entrypoint.sh is only available in this step for easier test handling with variables. FROM deps AS tests +ARG FEATURES +ENV FEATURES=${FEATURES} + # Skip IPv6 tests by default, as some CI environment don't have IPv6 available ARG ZEBRA_SKIP_IPV6_TESTS ENV ZEBRA_SKIP_IPV6_TESTS=${ZEBRA_SKIP_IPV6_TESTS:-1} -# Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh, -# separately from the test and production image builds. -ARG FEATURES -ARG TEST_FEATURES -ARG EXPERIMENTAL_FEATURES -# TODO: add empty $EXPERIMENTAL_FEATURES when we can avoid adding an extra space to the end of the string -ARG ENTRYPOINT_FEATURES="${FEATURES} ${TEST_FEATURES}" +# Set up the test environment the same way the production environment is. This +# is not very DRY as the same code repeats for the `runtime` target below, but I +# didn't find a suitable way to share the setup between the two targets. + +ENV UID=101 +ENV GID=${UID} +ENV USER="zebra" +ENV HOME="/home/${USER}" +ENV CARGO_HOME="${HOME}/.cargo/" + +RUN adduser --system --gid ${GID} --uid ${UID} --home ${HOME} ${USER} + +WORKDIR ${HOME} # Build Zebra test binaries, but don't run them # Leverage a cache mount to /usr/local/cargo/registry/ # for downloaded dependencies, a cache mount to /usr/local/cargo/git/db -# for git repository dependencies, and a cache mount to ${APP_HOME}/target/ for +# for git repository dependencies, and a cache mount to ${HOME}/target/ for # compiled dependencies which will speed up subsequent builds. # Leverage a bind mount to each crate directory to avoid having to copy the # source code into the container. Once built, copy the executable to an -# output directory before the cache mounted ${APP_HOME}/target/ is unmounted. +# output directory before the cache mounted ${HOME}/target/ is unmounted. RUN --mount=type=bind,source=zebrad,target=zebrad \ --mount=type=bind,source=zebra-chain,target=zebra-chain \ --mount=type=bind,source=zebra-network,target=zebra-network \ @@ -113,25 +99,38 @@ RUN --mount=type=bind,source=zebrad,target=zebrad \ --mount=type=bind,source=tower-fallback,target=tower-fallback \ --mount=type=bind,source=Cargo.toml,target=Cargo.toml \ --mount=type=bind,source=Cargo.lock,target=Cargo.lock \ - --mount=type=cache,target=${APP_HOME}/target/ \ + --mount=type=cache,target=${HOME}/target/ \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry/ \ -cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace --no-run && \ -cp ${APP_HOME}/target/release/zebrad /usr/local/bin && \ -cp ${APP_HOME}/target/release/zebra-checkpoints /usr/local/bin + cargo test --locked --release --workspace --no-run \ + --features "${FEATURES} zebra-checkpoints" && \ + cp ${HOME}/target/release/zebrad /usr/local/bin && \ + cp ${HOME}/target/release/zebra-checkpoints /usr/local/bin # Copy the lightwalletd binary and source files to be able to run tests COPY --from=electriccoinco/lightwalletd:latest /usr/local/bin/lightwalletd /usr/local/bin/ -COPY ./ ./ -# Entrypoint environment variables -ENV ENTRYPOINT_FEATURES=${ENTRYPOINT_FEATURES} -# We repeat the ARGs here, so they are available in the entrypoint.sh script for $RUN_ALL_EXPERIMENTAL_TESTS -ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" -ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" +# Use the same default config as in the production environment. +ENV ZEBRA_CONF_PATH="${HOME}/.config/zebrad.toml" +COPY --chown=${UID}:${GID} ./docker/default_zebra_config.toml ${ZEBRA_CONF_PATH} + +ARG LWD_CACHE_DIR +ENV LWD_CACHE_DIR="${HOME}/.cache/lwd" +RUN mkdir -p ${LWD_CACHE_DIR} +RUN chown -R ${UID}:${GID} ${LWD_CACHE_DIR} + +# Use the same cache dir as in the production environment. +ARG ZEBRA_CACHE_DIR +ENV ZEBRA_CACHE_DIR="${HOME}/.cache/zebra" +RUN mkdir -p ${ZEBRA_CACHE_DIR} +RUN chown -R ${UID}:${GID} ${ZEBRA_CACHE_DIR} -# By default, runs the entrypoint tests specified by the environmental variables (if any are set) -ENTRYPOINT [ "/etc/zebrad/entrypoint.sh" ] +COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY ./ ${HOME} + +RUN chown -R ${UID}:${GID} ${HOME} + +ENTRYPOINT [ "entrypoint.sh", "test" ] # In this stage we build a release (generate the zebrad binary) # @@ -161,68 +160,67 @@ RUN --mount=type=bind,source=tower-batch-control,target=tower-batch-control \ --mount=type=cache,target=${APP_HOME}/target/ \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry/ \ -cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad && \ -cp ${APP_HOME}/target/release/zebrad /usr/local/bin + cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad && \ + cp ${APP_HOME}/target/release/zebrad /usr/local/bin -# This stage is only used when deploying nodes or when only the resulting zebrad binary is needed -# -# To save space, this step starts from scratch using debian, and only adds the resulting -# binary from the `release` stage +# This step starts from scratch using Debian and only adds the resulting binary +# from the `release` stage. FROM debian:bookworm-slim AS runtime -# Set the default path for the zebrad binary -ARG APP_HOME -ENV APP_HOME=${APP_HOME} -WORKDIR ${APP_HOME} - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - rocksdb-tools \ - gosu \ - && rm -rf /var/lib/apt/lists/* /tmp/* +COPY --from=release /usr/local/bin/zebrad /usr/local/bin/ +COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh -# Create a non-privileged user that the app will run under. -# Running as root inside the container is running as root in the Docker host -# If an attacker manages to break out of the container, they will have root access to the host -# See https://docs.docker.com/go/dockerfile-user-best-practices/ -ARG USER=zebra +ARG FEATURES +ENV FEATURES=${FEATURES} + +# Create a non-privileged system user for running `zebrad`. +ARG USER="zebra" ENV USER=${USER} -ARG UID=10001 + +# System users have no home dirs, but we set one for users' convenience. +ARG HOME="/home/zebra" +WORKDIR ${HOME} + +# System UIDs should be set according to +# https://refspecs.linuxfoundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/uidrange.html. +# +# In Debian, the default dynamic range for system UIDs is defined by +# [FIRST_SYSTEM_UID, LAST_SYSTEM_UID], which is set to [100, 999] in +# `etc/adduser.conf`: +# https://manpages.debian.org/bullseye/adduser/adduser.8.en.html +# +# Debian assigns GID 100 to group `users`, so we set UID = GID = 101 as the +# default value. +ARG UID=101 ENV UID=${UID} -ARG GID=10001 +ARG GID=${UID} ENV GID=${GID} -RUN addgroup --system --gid ${GID} ${USER} \ - && adduser \ - --system \ - --disabled-login \ - --shell /bin/bash \ - --home ${APP_HOME} \ - --uid "${UID}" \ - --gid "${GID}" \ - ${USER} - -# Config settings for zebrad -ARG FEATURES -ENV FEATURES=${FEATURES} +RUN addgroup --system --gid ${GID} ${USER} +RUN adduser --system --gid ${GID} --uid ${UID} --home ${HOME} ${USER} -# Path and name of the config file -# These are set to a default value when not defined in the environment -ENV ZEBRA_CONF_DIR=${ZEBRA_CONF_DIR:-/etc/zebrad} -ENV ZEBRA_CONF_FILE=${ZEBRA_CONF_FILE:-zebrad.toml} +# We set the default locations of the conf and cache dirs according to the XDG +# spec: https://specifications.freedesktop.org/basedir-spec/latest/ -RUN mkdir -p ${ZEBRA_CONF_DIR} && chown ${UID}:${UID} ${ZEBRA_CONF_DIR} \ - && chown ${UID}:${UID} ${APP_HOME} +ARG ZEBRA_CONF_PATH="${HOME}/.config/zebrad.toml" +ENV ZEBRA_CONF_PATH=${ZEBRA_CONF_PATH} +COPY --chown=${UID}:${GID} ./docker/default_zebra_config.toml ${ZEBRA_CONF_PATH} -COPY --from=release /usr/local/bin/zebrad /usr/local/bin -COPY --from=release /etc/zebrad/entrypoint.sh /etc/zebrad +ARG ZEBRA_CACHE_DIR="${HOME}/.cache/zebra" +ENV ZEBRA_CACHE_DIR=${ZEBRA_CACHE_DIR} +RUN mkdir -p ${ZEBRA_CACHE_DIR} && chown -R ${UID}:${GID} ${ZEBRA_CACHE_DIR} -# Expose configured ports -EXPOSE 8233 18233 +RUN chown -R ${UID}:${GID} ${HOME} +USER $USER -# Update the config file based on the Docker run variables, -# and launch zebrad with it -ENTRYPOINT [ "/etc/zebrad/entrypoint.sh" ] +ENTRYPOINT [ "entrypoint.sh" ] CMD ["zebrad"] + +# TODO: Add a `monitoring` stage +# +# This stage will be based on `runtime`, and initially: +# +# - run `zebrad` on Testnet +# - with mining enabled using S-nomp and `nheqminer`. +# +# We can add further functionality to this stage for further purposes. diff --git a/docker/default_zebra_config.toml b/docker/default_zebra_config.toml new file mode 100644 index 00000000000..d31a702ade4 --- /dev/null +++ b/docker/default_zebra_config.toml @@ -0,0 +1,59 @@ +# Default configuration file for running Zebra in Docker. +# +# This file is tailored for Zebra running in Docker. Do not use it with Zebra +# running directly on your localhost as some fields are adjusted specifically +# for Docker. +# +# You can use this file as a starting point for custom configuration. If you +# don't specify a field, Zebra will use its default value. +# +# The config format, including a complete list of sections and fields, is +# documented here: +# https://docs.rs/zebrad/latest/zebrad/config/struct.ZebradConfig.html + +[network] +network = "Mainnet" +listen_addr = "0.0.0.0" +cache_dir = "/home/zebra/.cache/zebra" + +[rpc] +# The RPC server is disabled by default. To enable it, uncomment one of the +# lines below and alternatively set your own port. + +# listen_addr = "0.0.0.0:8232" # Mainnet +# listen_addr = "0.0.0.0:18232" # Testnet + +cookie_dir = "/home/zebra/.cache/zebra" + +[state] +cache_dir = "/home/zebra/.cache/zebra" + +[tracing] +# Zebra uses colored output if it is attached to a terminal. To disable colors, +# set `use_color` to false. To enable colors even for non-terminal outputs, set +# `use_color` to `true` and uncomment the line below. + +# force_use_color = true +use_color = true + +# Logging to a file is disabled by default. To enable it, uncomment the line +# below and alternatively set your own path. + +# log_file = "/home/zebra/.local/state/zebrad.log" + +# Sending tracing events to systemd-journald is disabled by default. To enable +# it, uncomment the line below. + +# use_journald = true + +[metrics] +# Metrics via Prometheus are disabled by default. To enable them, uncomment the +# line below and alternatively set your own port. + +# endpoint_addr = "0.0.0.0:9999" # Prometheus + +[mining] +# If you are going to use Zebra as a backend for a mining pool, set your mining +# address. + +# miner_address = "your_mining_address" diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml index fac94e3f4db..b5730359bcb 100644 --- a/docker/docker-compose.test.yml +++ b/docker/docker-compose.test.yml @@ -14,10 +14,10 @@ services: memory: 16G # Change this to the command you want to run, respecting the entrypoint.sh # For example, to run the tests, use the following command: - # command: ["cargo", "test", "--locked", "--release", "--features", "${TEST_FEATURES}", "--package", "zebrad", "--test", "acceptance", "--", "--nocapture", "--include-ignored", "sync_large_checkpoints_"] + # command: ["cargo", "test", "--locked", "--release", "--features", "${FEATURES}", "--package", "zebrad", "--test", "acceptance", "--", "--nocapture", "--include-ignored", "sync_large_checkpoints_"] volumes: - - zebrad-cache:/var/cache/zebrad-cache - - lwd-cache:/var/cache/lwd-cache + - zebrad-cache:/home/zebra/.cache/zebra + - lwd-cache:/home/zebra/.cache/lwd ports: # Zebra uses the following inbound and outbound TCP ports - "8232:8232" # Opens an RPC endpoint (for wallet storing and mining) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 22359488de1..f9c71d6e0d4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -29,7 +29,7 @@ services: # gid: '2001' # Rust's container default group gid # mode: 0440 volumes: - - zebrad-cache:/var/cache/zebrad-cache + - zebrad-cache:/home/zebra/.cache/zebra ports: # Zebra uses the following default inbound and outbound TCP ports - "8233:8233" # Mainnet Network (for peer connections) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ccd09f43c33..45376df5b47 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,186 +1,119 @@ #!/usr/bin/env bash -# This script serves as the entrypoint for the Zebra Docker container. +# Entrypoint for running Zebra in Docker. # -# Description: -# This script serves as the primary entrypoint for the Docker container. Its main responsibilities include: -# 1. Environment Setup: Prepares the environment by setting various flags and parameters. -# 2. Configuration Management: Dynamically generates the `zebrad.toml` configuration file based on environment variables, ensuring the node starts with the desired settings. -# 3. Test Execution: Can run a series of tests to validate functionality based on specified environment variables. -# 4. Node Startup: Starts the node, allowing it to begin its operations. +# The main script logic is at the bottom. # +# ## Notes +# +# - `$ZEBRA_CONF_PATH` must point to a Zebra conf file writable by `$USER`. -# Exit if a command fails -set -e -# Exit if any command in a pipeline fails -set -o pipefail - -#### -# General Variables -# These variables are used to run the Zebra node. -#### - -# Path and name of the config file. These two have defaults set in the Dockerfile. -: "${ZEBRA_CONF_DIR:=}" -: "${ZEBRA_CONF_FILE:=}" -# [network] -: "${NETWORK:=Mainnet}" -: "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" -# [consensus] -: "${ZEBRA_CHECKPOINT_SYNC:=true}" -# [state] -# Set this to change the default cached state directory -: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" -: "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd-cache}" -# [metrics] -: "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" -: "${METRICS_ENDPOINT_PORT:=9999}" -# [tracing] -: "${LOG_COLOR:=false}" -: "${TRACING_ENDPOINT_ADDR:=0.0.0.0}" -: "${TRACING_ENDPOINT_PORT:=3000}" -# [rpc] -: "${RPC_LISTEN_ADDR:=0.0.0.0}" -# if ${RPC_PORT} is not set, use the default value for the current network -if [[ -z "${RPC_PORT}" ]]; then - if [[ "${NETWORK}" = "Mainnet" ]]; then - : "${RPC_PORT:=8232}" - elif [[ "${NETWORK}" = "Testnet" ]]; then - : "${RPC_PORT:=18232}" - fi -fi +set -eo pipefail -#### -# Test Variables -# These variables are used to run tests in the Dockerfile. -#### - -: "${RUN_ALL_TESTS:=}" -: "${RUN_ALL_EXPERIMENTAL_TESTS:=}" -: "${TEST_FAKE_ACTIVATION_HEIGHTS:=}" -: "${TEST_ZEBRA_EMPTY_SYNC:=}" -: "${TEST_LWD_INTEGRATION:=}" -: "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES:=}" -: "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES:=}" -: "${TEST_DISK_REBUILD:=}" -: "${TEST_UPDATE_SYNC:=}" -: "${TEST_CHECKPOINT_SYNC:=}" -: "${GENERATE_CHECKPOINTS_MAINNET:=}" -: "${GENERATE_CHECKPOINTS_TESTNET:=}" -: "${TEST_LWD_RPC_CALL:=}" -: "${TEST_LWD_FULL_SYNC:=}" -: "${TEST_LWD_UPDATE_SYNC:=}" -: "${TEST_LWD_GRPC:=}" -: "${TEST_LWD_TRANSACTIONS:=}" -: "${TEST_GET_BLOCK_TEMPLATE:=}" -: "${TEST_SUBMIT_BLOCK:=}" -: "${ENTRYPOINT_FEATURES:=}" -: "${TEST_SCAN_TASK_COMMANDS:=}" - -# Configuration file path -if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]] && [[ -z "${ZEBRA_CONF_PATH}" ]]; then - ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" +# Exit early if `ZEBRA_CONF_PATH` does not point to a file. +if [[ ! -f "${ZEBRA_CONF_PATH}" ]]; then + echo "the ZEBRA_CONF_PATH env var does not point to a Zebra conf file" + exit 1 fi -# Populate `zebrad.toml` before starting zebrad, using the environmental -# variables set by the Dockerfile or the user. If the user has already created a config, don't replace it. +# Populates the config file for Zebra, using the env vars set by the Dockerfile +# or user. +# +# Also prints the content of the generated config file. +# +# ## Positional Parameters # -# We disable most ports by default, so the default config is secure. -# Users have to opt-in to additional functionality by setting environmental variables. -if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then - # Create the conf path and file - (mkdir -p "$(dirname "${ZEBRA_CONF_PATH}")" && touch "${ZEBRA_CONF_PATH}") || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } - # Populate the conf file - cat < "${ZEBRA_CONF_PATH}" -[network] -network = "${NETWORK}" -listen_addr = "${ZEBRA_LISTEN_ADDR}" -[state] -cache_dir = "${ZEBRA_CACHED_STATE_DIR}" -EOF - - if [[ " ${FEATURES} " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" -[metrics] -endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" -EOF +# - "$1": the file to write the config to +prepare_conf_file() { + # Set a custom `network`. + if [[ "${NETWORK}" ]]; then + sed -i '/network = ".*"/s/".*"/"'"${NETWORK//\"/}"'"/' "${1}" fi - if [[ -n "${RPC_PORT}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" -[rpc] -listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" -EOF + # Enable the RPC server by setting its port. + if [[ "${ZEBRA_RPC_PORT}" ]]; then + sed -i '/# listen_addr = "0.0.0.0:18232" # Testnet/d' "${1}" + sed -i 's/ *# Mainnet$//' "${1}" + sed -i '/# listen_addr = "0.0.0.0:8232"/s/^# //; s/8232/'"${ZEBRA_RPC_PORT//\"/}"'/' "${1}" fi - if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" -[tracing] -EOF - if [[ " ${FEATURES} " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" -endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" -EOF - fi - # Set this to log to a file, if not set, logs to standard output - if [[ -n "${LOG_FILE}" ]]; then - mkdir -p "$(dirname "${LOG_FILE}")" - cat <> "${ZEBRA_CONF_PATH}" -log_file = "${LOG_FILE}" -EOF - fi - # Zebra automatically detects if it is attached to a terminal, and uses colored output. - # Set this to 'true' to force using color even if the output is not a terminal. - # Set this to 'false' to disable using color even if the output is a terminal. - if [[ "${LOG_COLOR}" = "true" ]]; then - cat <> "${ZEBRA_CONF_PATH}" -force_use_color = true -EOF - elif [[ "${LOG_COLOR}" = "false" ]]; then - cat <> "${ZEBRA_CONF_PATH}" -use_color = false -EOF - fi + # Set a custom state, network and cookie cache dirs. + # + # We're pointing all three cache dirs at the same location, so users will find + # all cached data in that single location. We can introduce more env vars and + # use them to set the cache dirs separately if needed. + if [[ "${ZEBRA_CACHE_DIR}" ]]; then + mkdir -p "${ZEBRA_CACHE_DIR//\"/}" + sed -i 's|_dir = ".*"|_dir = "'"${ZEBRA_CACHE_DIR//\"/}"'"|' "${1}" fi - if [[ -n "${MINER_ADDRESS}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" -[mining] -miner_address = "${MINER_ADDRESS}" -EOF + # Enable the Prometheus metrics endpoint. + if [[ "${FEATURES}" == *"prometheus"* ]]; then + sed -i '/# endpoint_addr = "0.0.0.0:9999" # Prometheus/s/^# //' "${1}" fi -fi -if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then - # Print the config file - echo "Using zebrad.toml:" - cat "${ZEBRA_CONF_PATH}" -fi + # Enable logging to a file by setting a custom log file path. + if [[ "${LOG_FILE}" ]]; then + mkdir -p "$(dirname "${LOG_FILE//\"/}")" + sed -i 's|# log_file = ".*"|log_file = "'"${LOG_FILE//\"/}"'"|' "${1}" + fi + + # Enable or disable colored logs. + if [[ "${LOG_COLOR}" ]]; then + sed -i '/# force_use_color = true/s/^# //' "${1}" + sed -i '/use_color = true/s/true/'"${LOG_COLOR//\"/}"'/' "${1}" + fi + + # Enable or disable logging to systemd-journald. + if [[ "${USE_JOURNALD}" ]]; then + sed -i '/# use_journald = true/s/^# //; s/true/'"${USE_JOURNALD//\"/}"'/' "${1}" + fi + + # Set a mining address. + if [[ "${MINER_ADDRESS}" ]]; then + sed -i '/# miner_address = ".*"/{s/^# //; s/".*"/"'"${MINER_ADDRESS//\"/}"'"/}' "${1}" + fi + + # Trim all comments and empty lines. + sed -i '/^#/d; /^$/d' "${1}" + + echo "Prepared the following Zebra config:" + cat "$1" +} -# Function to list directory +# Checks if a directory contains subdirectories +# +# Exits with 0 if it does, and 1 otherwise. check_directory_files() { local dir="$1" # Check if the directory exists if [[ -d "${dir}" ]]; then # Check if there are any subdirectories if find "${dir}" -mindepth 1 -type d | read -r; then - # Subdirectories exist, so we continue : else - # No subdirectories, print message and exit with status 1 echo "No subdirectories found in ${dir}." exit 1 fi else - # Directory doesn't exist, print message and exit with status 1 echo "Directory ${dir} does not exist." exit 1 fi } -# Function to run cargo test with an arbitrary number of arguments +# Runs cargo test with an arbitrary number of arguments. +# +# ## Positional Parameters +# +# - '$1' must contain +# - either cargo FEATURES as described here: +# https://doc.rust-lang.org/cargo/reference/features.html#command-line-feature-options, +# - or be empty. +# - The remaining params will be appended to a command starting with +# `exec cargo test ... -- ...` run_cargo_test() { - # Start constructing the command, ensuring that $1 is enclosed in single quotes as it's a feature list + # Start constructing the command, ensuring that $1 is enclosed in single + # quotes as it's a feature list local cmd="exec cargo test --locked --release --features '$1' --package zebrad --test acceptance -- --nocapture --include-ignored" # Shift the first argument, as it's already included in the cmd @@ -194,170 +127,161 @@ run_cargo_test() { fi done - # Run the command using eval, this will replace the current process with the cargo command - eval "${cmd}" || { echo "Cargo test failed"; exit 1; } + # Run the command using eval. This will replace the current process with the + # cargo command. + echo "Running:" + echo "${cmd}" + eval "${cmd}" || { + echo "Cargo test failed" + exit 1 + } } -# Main Execution Logic: -# This script orchestrates the execution flow based on the provided arguments and environment variables. -# - If "$1" is '--', '-', or 'zebrad', the script processes the subsequent arguments for the 'zebrad' command. -# - If ENTRYPOINT_FEATURES is unset, it checks for ZEBRA_CONF_PATH. If set, 'zebrad' runs with this custom configuration; otherwise, it runs with the provided arguments. -# - If "$1" is an empty string and ENTRYPOINT_FEATURES is set, the script enters the testing phase, checking various environment variables to determine the specific tests to run. -# - Different tests or operations are triggered based on the respective conditions being met. -# - If "$1" doesn't match any of the above, it's assumed to be a command, which is executed directly. -# This structure ensures a flexible execution strategy, accommodating various scenarios such as custom configurations, different testing phases, or direct command execution. +# Runs tests depending on the env vars. +# +# Positional Parameters +# +# - $@: Arbitrary command that will be executed if no test env var is set. +run_tests() { + if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then + # Run unit, basic acceptance tests, and ignored tests, only showing command + # output if the test fails. If the lightwalletd environment variables are + # set, we will also run those tests. + exec cargo test --locked --release --workspace --features "${FEATURES}" \ + -- --nocapture --include-ignored --skip check_no_git_refs_in_cargo_lock + + elif [[ "${RUN_CHECK_NO_GIT_REFS}" -eq "1" ]]; then + # Run the check_no_git_refs_in_cargo_lock test. + exec cargo test --locked --release --workspace --features "${FEATURES}" \ + -- --nocapture --include-ignored check_no_git_refs_in_cargo_lock + + elif [[ "${TEST_FAKE_ACTIVATION_HEIGHTS}" -eq "1" ]]; then + # Run state tests with fake activation heights. + exec cargo test --locked --release --lib --features "zebra-test" \ + --package zebra-state \ + -- --nocapture --include-ignored with_fake_activation_heights + + elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then + # Test the scanner. + exec cargo test --locked --release --package zebra-scan \ + -- --nocapture --include-ignored scan_task_commands scan_start_where_left + + elif [[ "${TEST_ZEBRA_EMPTY_SYNC}" -eq "1" ]]; then + # Test that Zebra syncs and checkpoints a few thousand blocks from an empty + # state. + run_cargo_test "${FEATURES}" "sync_large_checkpoints_" + + elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on mainnet. + run_cargo_test "${FEATURES}" "full_sync_mainnet" + + elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on testnet. + run_cargo_test "${FEATURES}" "full_sync_testnet" + + elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then + # Run a Zebra sync up to the mandatory checkpoint. + run_cargo_test "${FEATURES} test_sync_to_mandatory_checkpoint_${NETWORK,,}" \ + "sync_to_mandatory_checkpoint_${NETWORK,,}" + echo "ran test_disk_rebuild" + + elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached tip, and syncing to the latest + # tip. + run_cargo_test "${FEATURES}" "zebrad_update_sync" + + elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing + # past it. + run_cargo_test "${FEATURES} test_sync_past_mandatory_checkpoint_${NETWORK,,}" \ + "sync_past_mandatory_checkpoint_${NETWORK,,}" + + elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra from a cached state on mainnet. + # + # TODO: disable or filter out logs like: + # test generate_checkpoints_mainnet has been running for over 60 seconds + run_cargo_test "${FEATURES}" "generate_checkpoints_mainnet" + + elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra on testnet. + # + # This test might fail if testnet is unstable. + run_cargo_test "${FEATURES}" "generate_checkpoints_testnet" + + elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then + # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. + # Run both the fully synced RPC test and the subtree snapshot test, one test + # at a time. Since these tests use the same cached state, a state problem in + # the first test can fail the second test. + run_cargo_test "${FEATURES}" "--test-threads" "1" "fully_synced_rpc_" + + elif [[ "${TEST_LWD_INTEGRATION}" -eq "1" ]]; then + # Test launching lightwalletd with an empty lightwalletd and Zebra state. + run_cargo_test "${FEATURES}" "lightwalletd_integration" + + elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then + # Starting at a cached Zebra tip, run a lightwalletd sync to tip. + run_cargo_test "${FEATURES}" "lightwalletd_full_sync" + + elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. + run_cargo_test "${FEATURES}" "lightwalletd_update_sync" + + # These tests actually use gRPC. + elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to + # lightwalletd, which calls Zebra. + run_cargo_test "${FEATURES}" "lightwalletd_wallet_grpc_tests" + + elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test sending + # transactions gRPC call to lightwalletd, which calls Zebra. + run_cargo_test "${FEATURES}" "sending_transactions_using_lightwalletd" + + # These tests use mining code, but don't use gRPC. + elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test getting a block template from + # Zebra's RPC server. + run_cargo_test "${FEATURES}" "get_block_template" + + elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test sending a block to Zebra's RPC + # port. + run_cargo_test "${FEATURES}" "submit_block" -case "$1" in - --* | -* | zebrad) - shift - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad "$@" || { echo "Execution failed"; exit 1; } - fi - ;; - "") - if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then - # Validate the test variables - # For these tests, we activate the test features to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then - # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails. - # If the lightwalletd environmental variables are set, we will also run those tests. - exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored --skip check_no_git_refs_in_cargo_lock - - elif [[ "${RUN_ALL_EXPERIMENTAL_TESTS}" -eq "1" ]]; then - # Run unit, basic acceptance tests, and ignored tests with experimental features. - # If the lightwalletd environmental variables are set, we will also run those tests. - exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES_EXPERIMENTAL}" --workspace -- --nocapture --include-ignored --skip check_no_git_refs_in_cargo_lock - - elif [[ "${RUN_CHECK_NO_GIT_REFS}" -eq "1" ]]; then - # Run the check_no_git_refs_in_cargo_lock test. - exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored check_no_git_refs_in_cargo_lock - - elif [[ "${TEST_FAKE_ACTIVATION_HEIGHTS}" -eq "1" ]]; then - # Run state tests with fake activation heights. - exec cargo test --locked --release --features "zebra-test" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights - - elif [[ "${TEST_ZEBRA_EMPTY_SYNC}" -eq "1" ]]; then - # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. - run_cargo_test "${ENTRYPOINT_FEATURES}" "sync_large_checkpoints_" - - elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on mainnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" - # List directory generated by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on testnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" - # List directory generated by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then - # Run a Zebra sync up to the mandatory checkpoint. - # - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" - - elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" - - elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra from a cached state on mainnet. - # - # TODO: disable or filter out logs like: - # test generate_checkpoints_mainnet has been running for over 60 seconds - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" - - elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra on testnet. - # - # This test might fail if testnet is unstable. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" - - elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then - # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. - # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads" "1" "fully_synced_rpc_" - - elif [[ "${TEST_LWD_INTEGRATION}" -eq "1" ]]; then - # Test launching lightwalletd with an empty lightwalletd and Zebra state. - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_integration" - - elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then - # Starting at a cached Zebra tip, run a lightwalletd sync to tip. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - - elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" - - # These tests actually use gRPC. - elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" - - elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" - - # These tests use mining code, but don't use gRPC. - elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "get_block_template" - - elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block" - - elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then - # Test that the scan task commands are working. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands - - else - exec "$@" - fi - fi - ;; - *) - if command -v gosu >/dev/null 2>&1; then - exec gosu "$USER" "$@" + else + if [[ "$1" == "zebrad" ]]; then + shift + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" else exec "$@" fi - ;; + fi +} + +# Main Script Logic + +prepare_conf_file "$ZEBRA_CONF_PATH" + +# - If "$1" is "--", "-", or "zebrad", run `zebrad` with the remaining params. +# - If "$1" is "tests", run tests. +# - TODO: If "$1" is "monitoring", start a monitoring node. +# - If "$1" doesn't match any of the above, run "$@" directly. +case "$1" in +--* | -* | zebrad) + shift + exec zebrad --config "${ZEBRA_CONF_PATH}" "$@" + ;; +test) + shift + run_tests "$@" + ;; +monitoring) + # TODO: Impl logic for starting a monitoring node. + : + ;; +*) + exec "$@" + ;; esac diff --git a/zebra-scan/tests/scan_task_commands.rs b/zebra-scan/tests/scan_task_commands.rs index 20c4edfe757..1f38a3302fe 100644 --- a/zebra-scan/tests/scan_task_commands.rs +++ b/zebra-scan/tests/scan_task_commands.rs @@ -3,7 +3,7 @@ //! This test requires a cached chain state that is partially synchronized past the //! Sapling activation height and [`REQUIRED_MIN_TIP_HEIGHT`] //! -//! export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +//! export ZEBRA_CACHE_DIR="/path/to/zebra/state" //! cargo test scan_task_commands --features="shielded-scan" -- --ignored --nocapture #![allow(dead_code, non_local_definitions)] @@ -59,9 +59,9 @@ pub(crate) async fn run() -> Result<()> { // This is currently needed for the 'Check startup logs' step in CI to pass. tracing::info!("Zcash network: {network}"); - let zebrad_state_path = match std::env::var_os("ZEBRA_CACHED_STATE_DIR") { + let zebrad_state_path = match std::env::var_os("ZEBRA_CACHE_DIR") { None => { - tracing::error!("ZEBRA_CACHED_STATE_DIR is not set"); + tracing::error!("ZEBRA_CACHE_DIR is not set"); return Ok(()); } Some(path) => std::path::PathBuf::from(path), diff --git a/zebra-scan/tests/scanner.rs b/zebra-scan/tests/scanner.rs index 3708caa07a9..9423119fb84 100644 --- a/zebra-scan/tests/scanner.rs +++ b/zebra-scan/tests/scanner.rs @@ -132,7 +132,7 @@ async fn scan_binary_starts() -> Result<()> { /// /// Needs a cache state close to the tip. A possible way to run it locally is: /// -/// export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +/// export ZEBRA_CACHE_DIR="/path/to/zebra/state" /// cargo test scan_start_where_left -- --ignored --nocapture /// /// The test will run zebrad with a key to scan, scan the first few blocks after sapling and then stops. @@ -145,9 +145,9 @@ async fn scan_start_where_left() -> Result<()> { let _init_guard = zebra_test::init(); - let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHED_STATE_DIR") else { + let Ok(zebrad_cachedir) = std::env::var("ZEBRA_CACHE_DIR") else { tracing::info!("skipping scan_start_where_left test due to missing cached state, \ - please set a ZEBRA_CACHED_STATE_DIR env var with a populated and valid path to run this test"); + please set a ZEBRA_CACHE_DIR env var with a populated and valid path to run this test"); return Ok(()); }; @@ -244,7 +244,7 @@ async fn scan_start_where_left() -> Result<()> { /// Example of how to run the scan_task_commands test locally: /// /// ```console -/// RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture +/// RUST_LOG=info ZEBRA_CACHE_DIR=/path/to/zebra/state cargo test scan_task_commands -- --include-ignored --nocapture /// ``` #[tokio::test] #[ignore] diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 270a0e955c7..89afff9bbcc 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -30,7 +30,7 @@ //! will allow this test to run or give up. Value for the Mainnet full sync tests. //! - `FULL_SYNC_TESTNET_TIMEOUT_MINUTES` env variable: The total number of minutes we //! will allow this test to run or give up. Value for the Testnet full sync tests. -//! - `ZEBRA_CACHED_STATE_DIR` env variable: The path to a Zebra cached state directory. +//! - `ZEBRA_CACHE_DIR` env variable: The path to a Zebra cached state directory. //! If not set, it defaults to `/zebrad-cache`. For some sync tests, this directory needs to be //! created in the file system with write permissions. //! @@ -41,15 +41,15 @@ //! //! $ cargo test sync_large_checkpoints_mempool_mainnet -- --ignored --nocapture //! -//! $ export ZEBRA_CACHED_STATE_DIR="/zebrad-cache" -//! $ sudo mkdir -p "$ZEBRA_CACHED_STATE_DIR" -//! $ sudo chmod 777 "$ZEBRA_CACHED_STATE_DIR" +//! $ export ZEBRA_CACHE_DIR="/zebrad-cache" +//! $ sudo mkdir -p "$ZEBRA_CACHE_DIR" +//! $ sudo chmod 777 "$ZEBRA_CACHE_DIR" //! $ export FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 //! $ cargo test full_sync_mainnet -- --ignored --nocapture //! -//! $ export ZEBRA_CACHED_STATE_DIR="/zebrad-cache" -//! $ sudo mkdir -p "$ZEBRA_CACHED_STATE_DIR" -//! $ sudo chmod 777 "$ZEBRA_CACHED_STATE_DIR" +//! $ export ZEBRA_CACHE_DIR="/zebrad-cache" +//! $ sudo mkdir -p "$ZEBRA_CACHE_DIR" +//! $ sudo chmod 777 "$ZEBRA_CACHE_DIR" //! $ export FULL_SYNC_TESTNET_TIMEOUT_MINUTES=600 //! $ cargo test full_sync_testnet -- --ignored --nocapture //! ``` @@ -70,9 +70,9 @@ //! at least the `ZEBRA_TEST_LIGHTWALLETD` environment variable is present: //! //! - `ZEBRA_TEST_LIGHTWALLETD` env variable: Needs to be present to run any of the lightwalletd tests. -//! - `ZEBRA_CACHED_STATE_DIR` env variable: The path to a Zebra cached state directory. +//! - `ZEBRA_CACHE_DIR` env variable: The path to a Zebra cached state directory. //! If not set, it defaults to `/zebrad-cache`. -//! - `LIGHTWALLETD_DATA_DIR` env variable: The path to a lightwalletd database. +//! - `LWD_CACHE_DIR` env variable: The path to a lightwalletd database. //! - `--features lightwalletd-grpc-tests` cargo flag: The flag given to cargo to build the source code of the running test. //! //! Here are some examples of running each test: @@ -82,29 +82,29 @@ //! $ cargo test lightwalletd_integration -- --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" +//! $ export ZEBRA_CACHE_DIR="/path/to/zebra/state" +//! $ export LWD_CACHE_DIR="/path/to/lightwalletd/database" //! $ cargo test lightwalletd_update_sync -- --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +//! $ export ZEBRA_CACHE_DIR="/path/to/zebra/state" //! $ cargo test lightwalletd_full_sync -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true //! $ cargo test lightwalletd_test_suite -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" +//! $ export ZEBRA_CACHE_DIR="/path/to/zebra/state" //! $ cargo test fully_synced_rpc_test -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" +//! $ export ZEBRA_CACHE_DIR="/path/to/zebra/state" +//! $ export LWD_CACHE_DIR="/path/to/lightwalletd/database" //! $ cargo test sending_transactions_using_lightwalletd --features lightwalletd-grpc-tests -- --ignored --nocapture //! //! $ export ZEBRA_TEST_LIGHTWALLETD=true -//! $ export ZEBRA_CACHED_STATE_DIR="/path/to/zebra/state" -//! $ export LIGHTWALLETD_DATA_DIR="/path/to/lightwalletd/database" +//! $ export ZEBRA_CACHE_DIR="/path/to/zebra/state" +//! $ export LWD_CACHE_DIR="/path/to/lightwalletd/database" //! $ cargo test lightwalletd_wallet_grpc_tests --features lightwalletd-grpc-tests -- --ignored --nocapture //! ``` //! @@ -113,19 +113,19 @@ //! Example of how to run the get_block_template test: //! //! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture +//! ZEBRA_CACHE_DIR=/path/to/zebra/state cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture //! ``` //! //! Example of how to run the submit_block test: //! //! ```console -//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture +//! ZEBRA_CACHE_DIR=/path/to/zebra/state cargo test submit_block --features getblocktemplate-rpcs --release -- --ignored --nocapture //! ``` //! //! Example of how to run the has_spending_transaction_ids test: //! //! ```console -//! RUST_LOG=info ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state cargo test has_spending_transaction_ids --features "indexer" --release -- --ignored --nocapture +//! RUST_LOG=info ZEBRA_CACHE_DIR=/path/to/zebra/state cargo test has_spending_transaction_ids --features "indexer" --release -- --ignored --nocapture //! ``` //! //! Please refer to the documentation of each test for more information. @@ -134,8 +134,8 @@ //! //! Generate checkpoints on mainnet and testnet using a cached state: //! ```console -//! GENERATE_CHECKPOINTS_MAINNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh -//! GENERATE_CHECKPOINTS_TESTNET=1 ENTRYPOINT_FEATURES=zebra-checkpoints ZEBRA_CACHED_STATE_DIR=/path/to/zebra/state docker/entrypoint.sh +//! GENERATE_CHECKPOINTS_MAINNET=1 FEATURES=zebra-checkpoints ZEBRA_CACHE_DIR=/path/to/zebra/state docker/entrypoint.sh +//! GENERATE_CHECKPOINTS_TESTNET=1 FEATURES=zebra-checkpoints ZEBRA_CACHE_DIR=/path/to/zebra/state docker/entrypoint.sh //! ``` //! //! ## Disk Space for Testing @@ -1279,7 +1279,7 @@ fn full_sync_test(network: Network, timeout_argument_name: &str) -> Result<()> { // // Replace hard-coded values in create_cached_database_height with: // - the timeout in the environmental variable - // - the path from ZEBRA_CACHED_STATE_DIR + // - the path from ZEBRA_CACHE_DIR if let Some(_timeout_minutes) = timeout_argument { create_cached_database_height( &network, @@ -1792,7 +1792,7 @@ fn lightwalletd_integration() -> Result<()> { /// Make sure `zebrad` can sync from peers, but don't actually launch `lightwalletd`. /// -/// This test only runs when the `ZEBRA_CACHED_STATE_DIR` env var is set. +/// This test only runs when the `ZEBRA_CACHE_DIR` env var is set. /// /// This test might work on Windows. #[test] @@ -1803,8 +1803,8 @@ fn zebrad_update_sync() -> Result<()> { /// Make sure `lightwalletd` can sync from Zebra, in update sync mode. /// /// This test only runs when: -/// - the `ZEBRA_TEST_LIGHTWALLETD`, `ZEBRA_CACHED_STATE_DIR`, and -/// `LIGHTWALLETD_DATA_DIR` env vars are set, and +/// - the `ZEBRA_TEST_LIGHTWALLETD`, `ZEBRA_CACHE_DIR`, and +/// `LWD_CACHE_DIR` env vars are set, and /// - Zebra is compiled with `--features=lightwalletd-grpc-tests`. /// /// This test doesn't work on Windows, so it is always skipped on that platform. @@ -1818,7 +1818,7 @@ fn lightwalletd_update_sync() -> Result<()> { /// Make sure `lightwalletd` can fully sync from genesis using Zebra. /// /// This test only runs when: -/// - the `ZEBRA_TEST_LIGHTWALLETD` and `ZEBRA_CACHED_STATE_DIR` env vars are set, and +/// - the `ZEBRA_TEST_LIGHTWALLETD` and `ZEBRA_CACHE_DIR` env vars are set, and /// - Zebra is compiled with `--features=lightwalletd-grpc-tests`. /// /// @@ -1837,9 +1837,9 @@ fn lightwalletd_full_sync() -> Result<()> { /// /// Runs the tests in this order: /// - launch lightwalletd with empty states, -/// - if `ZEBRA_CACHED_STATE_DIR` is set: +/// - if `ZEBRA_CACHE_DIR` is set: /// - run a full sync -/// - if `ZEBRA_CACHED_STATE_DIR` and `LIGHTWALLETD_DATA_DIR` are set: +/// - if `ZEBRA_CACHE_DIR` and `LWD_CACHE_DIR` are set: /// - run a quick update sync, /// - run a send transaction gRPC test, /// - run read-only gRPC tests. @@ -1855,7 +1855,7 @@ async fn lightwalletd_test_suite() -> Result<()> { launches_lightwalletd: true, })?; - // Only runs when ZEBRA_CACHED_STATE_DIR is set. + // Only runs when ZEBRA_CACHE_DIR is set. lightwalletd_integration_test(UpdateZebraCachedStateNoRpc)?; // These tests need the compile-time gRPC feature @@ -1863,21 +1863,21 @@ async fn lightwalletd_test_suite() -> Result<()> { { // Do the quick tests first - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + // Only runs when LWD_CACHE_DIR and ZEBRA_CACHE_DIR are set lightwalletd_integration_test(UpdateCachedState)?; - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + // Only runs when LWD_CACHE_DIR and ZEBRA_CACHE_DIR are set common::lightwalletd::wallet_grpc_test::run().await?; // Then do the slow tests - // Only runs when ZEBRA_CACHED_STATE_DIR is set. + // Only runs when ZEBRA_CACHE_DIR is set. // When manually running the test suite, allow cached state in the full sync test. lightwalletd_integration_test(FullSyncFromGenesis { allow_lightwalletd_cached_state: true, })?; - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + // Only runs when LWD_CACHE_DIR and ZEBRA_CACHE_DIR are set common::lightwalletd::send_transaction_test::run().await?; } diff --git a/zebrad/tests/common/cached_state.rs b/zebrad/tests/common/cached_state.rs index c290cde2cd9..a03fafb6f1b 100644 --- a/zebrad/tests/common/cached_state.rs +++ b/zebrad/tests/common/cached_state.rs @@ -32,7 +32,7 @@ use crate::common::{ }; /// The environmental variable that holds the path to a directory containing a cached Zebra state. -pub const ZEBRA_CACHED_STATE_DIR: &str = "ZEBRA_CACHED_STATE_DIR"; +pub const ZEBRA_CACHE_DIR: &str = "ZEBRA_CACHE_DIR"; /// In integration tests, the interval between database format checks for newly added blocks. /// diff --git a/zebrad/tests/common/configs/v2.1.0.toml b/zebrad/tests/common/configs/v2.1.0.toml new file mode 100644 index 00000000000..54613ba842a --- /dev/null +++ b/zebrad/tests/common/configs/v2.1.0.toml @@ -0,0 +1,85 @@ +# Default configuration for zebrad. +# +# This file can be used as a skeleton for custom configs. +# +# Unspecified fields use default values. Optional fields are Some(field) if the +# field is present and None if it is absent. +# +# This file is generated as an example using zebrad's current defaults. +# You should set only the config options you want to keep, and delete the rest. +# Only a subset of fields are present in the skeleton, since optional values +# whose default is None are omitted. +# +# The config format (including a complete list of sections and fields) is +# documented here: +# https://docs.rs/zebrad/latest/zebrad/config/struct.ZebradConfig.html +# +# zebrad attempts to load configs in the following order: +# +# 1. The -c flag on the command line, e.g., `zebrad -c myconfig.toml start`; +# 2. The file `zebrad.toml` in the users's preference directory (platform-dependent); +# 3. The default config. +# +# The user's preference directory and the default path to the `zebrad` config are platform dependent, +# based on `dirs::preference_dir`, see https://docs.rs/dirs/latest/dirs/fn.preference_dir.html : +# +# | Platform | Value | Example | +# | -------- | ------------------------------------- | ---------------------------------------------- | +# | Linux | `$XDG_CONFIG_HOME` or `$HOME/.config` | `/home/alice/.config/zebrad.toml` | +# | macOS | `$HOME/Library/Preferences` | `/Users/Alice/Library/Preferences/zebrad.toml` | +# | Windows | `{FOLDERID_RoamingAppData}` | `C:\Users\Alice\AppData\Local\zebrad.toml` | + +[consensus] +checkpoint_sync = true + +[mempool] +eviction_memory_time = "1h" +tx_cost_limit = 80000000 + +[metrics] + +[mining] +debug_like_zcashd = true + +[network] +cache_dir = true +crawl_new_peer_interval = "1m 1s" +initial_mainnet_peers = [ + "dnsseed.z.cash:8233", + "dnsseed.str4d.xyz:8233", + "mainnet.seeder.zfnd.org:8233", + "mainnet.is.yolo.money:8233", +] +initial_testnet_peers = [ + "dnsseed.testnet.z.cash:18233", + "testnet.seeder.zfnd.org:18233", + "testnet.is.yolo.money:18233", +] +listen_addr = "0.0.0.0:8233" +max_connections_per_ip = 1 +network = "Mainnet" +peerset_initial_target_size = 25 + +[rpc] +cookie_dir = "cache_dir" +debug_force_finished_sync = false +enable_cookie_auth = true +parallel_cpu_threads = 0 + +[state] +cache_dir = "cache_dir" +delete_old_database = true +ephemeral = false + +[sync] +checkpoint_verify_concurrency_limit = 1000 +download_concurrency_limit = 50 +full_verify_concurrency_limit = 20 +parallel_cpu_threads = 0 + +[tracing] +buffer_limit = 128000 +force_use_color = false +use_color = true +use_journald = false + diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 0cbb52e7cb7..d4ee17f323e 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -210,7 +210,7 @@ where /// Spawns a zebrad instance on `network` to test lightwalletd with `test_type`. /// /// If `use_internet_connection` is `false` then spawn, but without any peers. -/// This prevents it from downloading blocks. Instead, use the `ZEBRA_CACHED_STATE_DIR` +/// This prevents it from downloading blocks. Instead, use the `ZEBRA_CACHE_DIR` /// environmental variable to provide an initial state to the zebrad instance. /// /// Returns: @@ -261,7 +261,7 @@ pub fn spawn_zebrad_for_rpc + Debug>( /// Otherwise, just create an empty state in this test's new temporary directory. /// /// If `use_internet_connection` is `false` then spawn, but without any peers. -/// This prevents it from downloading blocks. Instead, use the `ZEBRA_CACHED_STATE_DIR` +/// This prevents it from downloading blocks. Instead, use the `ZEBRA_CACHE_DIR` /// environmental variable to provide an initial state to the zebrad instance. /// /// Returns: diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index fc10242d80f..376ce096865 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -50,7 +50,7 @@ pub const ZEBRA_TEST_LIGHTWALLETD: &str = "ZEBRA_TEST_LIGHTWALLETD"; /// /// Can also be used to speed up the [`sending_transactions_using_lightwalletd`] test, /// by skipping the lightwalletd initial sync. -pub const LIGHTWALLETD_DATA_DIR: &str = "LIGHTWALLETD_DATA_DIR"; +pub const LWD_CACHE_DIR: &str = "LWD_CACHE_DIR"; /// Should we skip Zebra lightwalletd integration tests? #[allow(clippy::print_stderr)] @@ -76,7 +76,7 @@ pub fn zebra_skip_lightwalletd_tests() -> bool { /// Spawns a lightwalletd instance on `network`, connected to `zebrad_rpc_address`, /// with its gRPC server functionality enabled. /// -/// Expects cached state based on the `test_type`. Use the `LIGHTWALLETD_DATA_DIR` +/// Expects cached state based on the `test_type`. Use the `LWD_CACHE_DIR` /// environmental variable to provide an initial state to the lightwalletd instance. /// /// Returns: diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index bac394099f5..cc8fd06f7fd 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -328,10 +328,10 @@ pub fn check_sync_logs_until( /// Returns the cache directory for Zebra's state. /// -/// It checks the `ZEBRA_CACHED_STATE_DIR` environment variable and returns its value if set. +/// It checks the `ZEBRA_CACHE_DIR` environment variable and returns its value if set. /// Otherwise, it defaults to `"/zebrad-cache"`. fn get_zebra_cached_state_dir() -> PathBuf { - env::var("ZEBRA_CACHED_STATE_DIR") + env::var("ZEBRA_CACHE_DIR") .unwrap_or_else(|_| "/zebrad-cache".to_string()) .into() } diff --git a/zebrad/tests/common/test_type.rs b/zebrad/tests/common/test_type.rs index 5621b12b690..cacfc55061d 100644 --- a/zebrad/tests/common/test_type.rs +++ b/zebrad/tests/common/test_type.rs @@ -14,14 +14,14 @@ use zebra_test::{command::NO_MATCHES_REGEX_ITER, prelude::*}; use zebrad::config::ZebradConfig; use super::{ - cached_state::ZEBRA_CACHED_STATE_DIR, + cached_state::ZEBRA_CACHE_DIR, config::{default_test_config, random_known_rpc_port_config}, failure_messages::{ LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES, LIGHTWALLETD_FAILURE_MESSAGES, PROCESS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES, }, launch::{LIGHTWALLETD_DELAY, LIGHTWALLETD_FULL_SYNC_TIP_DELAY, LIGHTWALLETD_UPDATE_TIP_DELAY}, - lightwalletd::LIGHTWALLETD_DATA_DIR, + lightwalletd::LWD_CACHE_DIR, sync::FINISH_PARTIAL_SYNC_TIMEOUT, }; @@ -140,7 +140,7 @@ impl TestType { } } - /// Can this test create a new `LIGHTWALLETD_DATA_DIR` cached state? + /// Can this test create a new `LWD_CACHE_DIR` cached state? pub fn can_create_lightwalletd_cached_state(&self) -> bool { match self { LaunchWithEmptyState { .. } | UseAnyState => false, @@ -152,13 +152,13 @@ impl TestType { /// Returns the Zebra state path for this test, if set. #[allow(clippy::print_stderr)] pub fn zebrad_state_path>(&self, test_name: S) -> Option { - match env::var_os(ZEBRA_CACHED_STATE_DIR) { + match env::var_os(ZEBRA_CACHE_DIR) { Some(path) => Some(path.into()), None => { let test_name = test_name.as_ref(); eprintln!( "skipped {test_name:?} {self:?} lightwalletd test, \ - set the {ZEBRA_CACHED_STATE_DIR:?} environment variable to run the test", + set the {ZEBRA_CACHE_DIR:?} environment variable to run the test", ); None @@ -235,24 +235,24 @@ impl TestType { if !self.launches_lightwalletd() || !use_or_create_lwd_cache { tracing::info!( "running {test_name:?} {self:?} lightwalletd test, \ - ignoring any cached state in the {LIGHTWALLETD_DATA_DIR:?} environment variable", + ignoring any cached state in the {LWD_CACHE_DIR:?} environment variable", ); return None; } - match env::var_os(LIGHTWALLETD_DATA_DIR) { + match env::var_os(LWD_CACHE_DIR) { Some(path) => Some(path.into()), None => { if self.needs_lightwalletd_cached_state() { tracing::info!( "skipped {test_name:?} {self:?} lightwalletd test, \ - set the {LIGHTWALLETD_DATA_DIR:?} environment variable to run the test", + set the {LWD_CACHE_DIR:?} environment variable to run the test", ); } else if self.allow_lightwalletd_cached_state() { tracing::info!( "running {test_name:?} {self:?} lightwalletd test without cached state, \ - set the {LIGHTWALLETD_DATA_DIR:?} environment variable to run with cached state", + set the {LWD_CACHE_DIR:?} environment variable to run with cached state", ); }