diff --git a/.github/workflows/benchmark-prs.yml b/.github/workflows/benchmark-prs.yml index eb27cf7ffc..ecd90a480c 100644 --- a/.github/workflows/benchmark-prs.yml +++ b/.github/workflows/benchmark-prs.yml @@ -43,7 +43,7 @@ jobs: # it will be better to execute bench test with `local`, # to make the measurement results reflect speed improvement or regression more accurately. - name: Build binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -71,7 +71,7 @@ jobs: - name: Start a client instance to compare memory usage shell: bash - run: ./target/release/ant --log-output-dest=data-dir file upload "./the-test-data.zip" + run: ./target/release/ant --log-output-dest=data-dir --local file upload "./the-test-data.zip" env: ANT_LOG: "all" timeout-minutes: 5 @@ -169,7 +169,7 @@ jobs: # # Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr, # # passes to tee which displays it in the terminal and writes to output.txt # run: | - # cargo criterion --features=local --message-format=json 2>&1 -p sn_cli | tee -a output.txt + # cargo criterion --message-format=json 2>&1 -p sn_cli | tee -a output.txt # cat output.txt | rg benchmark-complete | jq -s 'map({ # name: (.id | split("/"))[-1], # unit: "MiB/s", diff --git a/.github/workflows/cross-platform.yml b/.github/workflows/cross-platform.yml deleted file mode 100644 index 7b268cba02..0000000000 --- a/.github/workflows/cross-platform.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Cross platform checks - -on: - # tests must run for a PR to be valid and pass merge queue muster - # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors - # the merge run checks should show on master and enable this clear test/passing history - merge_group: - branches: [main] - pull_request: - branches: ["*"] - -env: - CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI. - -jobs: - - wasm: - if: "!startsWith(github.event.head_commit.message, 'chore(release):')" - name: wasm32-unknown-unknown builds - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - name: Install wasm-pack - run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - - name: Build WASM package - # --dev to avoid optimisation - run: wasm-pack build --dev --target=web autonomi - timeout-minutes: 30 - - - name: Cargo check for WASM - # Allow clippy lints (these can be pedantic on WASM), but deny regular Rust warnings - run: cargo clippy --target=wasm32-unknown-unknown --package=autonomi --lib --tests -- --allow=clippy::all --deny=warnings - timeout-minutes: 30 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..5edf449279 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,34 @@ +name: Deploy Documentation +on: + push: + branches: + - main + +permissions: + contents: write + +jobs: + deploy: + env: + GH_TOKEN: ${{ secrets.AUTONOMI_PAT }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mkdocs-material mkdocstrings mkdocstrings-python mkdocs-git-revision-date-localized-plugin + + - name: Deploy Documentation + run: | + git config --global user.name "github-actions" + git config --global user.email "github-actions@github.com" + mkdocs gh-deploy --force diff --git a/.github/workflows/generate-benchmark-charts.yml b/.github/workflows/generate-benchmark-charts.yml index 5ec91d7641..ac23f9a044 100644 --- a/.github/workflows/generate-benchmark-charts.yml +++ b/.github/workflows/generate-benchmark-charts.yml @@ -46,7 +46,7 @@ jobs: run: wget https://sn-node.s3.eu-west-2.amazonaws.com/the-test-data.zip - name: Build node and cli binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -67,7 +67,7 @@ jobs: # Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr, # passes to tee which displays it in the terminal and writes to output.txt run: | - cargo criterion --features=local --message-format=json 2>&1 -p autonomi-cli | tee -a output.txt + cargo criterion --message-format=json 2>&1 -p ant | tee -a output.txt cat output.txt | rg benchmark-complete | jq -s 'map({ name: (.id | split("/"))[-1], unit: "MiB/s", @@ -100,7 +100,7 @@ jobs: - name: Start a client instance to compare memory usage shell: bash - run: cargo run --bin ant --release -- --log-output-dest data-dir file upload the-test-data.zip + run: cargo run --bin ant --release -- --log-output-dest data-dir --local file upload the-test-data.zip env: ANT_LOG: "all" diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 89350fc49f..eff0181666 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -36,7 +36,7 @@ jobs: run: sudo apt-get install -y ripgrep - name: Build binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -70,7 +70,7 @@ jobs: shell: bash - name: File upload - run: ./target/release/ant --log-output-dest=data-dir file upload --public "./the-test-data.zip" > ./upload_output 2>&1 + run: ./target/release/ant --log-output-dest=data-dir --local file upload --public "./the-test-data.zip" > ./upload_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 15 @@ -98,8 +98,8 @@ jobs: ls -l $ANT_DATA_PATH/client_first/logs mkdir $ANT_DATA_PATH/client ls -l $ANT_DATA_PATH - ./target/release/ant --log-output-dest=data-dir file upload --public "./the-test-data.zip" > ./upload_output_second 2>&1 - rg 'Total cost: 0 AttoTokens' ./upload_output_second -c --stats + ./target/release/ant --log-output-dest=data-dir --local file upload --public "./the-test-data.zip" > ./upload_output_second 2>&1 + rg 'All chunks already exist on the network.' ./upload_output_second -c --stats env: ANT_LOG: "all" timeout-minutes: 25 @@ -150,9 +150,7 @@ jobs: if: always() - name: File Download - run: > - ./target/release/ant - --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources + run: ./target/release/ant --log-output-dest=data-dir --local file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources env: ANT_LOG: "v" timeout-minutes: 2 diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 9e1dadd148..9e766cb84e 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: wagoid/commitlint-github-action@7f0a61df502599e1f1f50880aaa7ec1e2c0592f2 + - uses: wagoid/commitlint-github-action@b948419dd99f3fd78a6548d48f94e3df7f6bf3ed checks: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" @@ -86,10 +86,6 @@ jobs: # resulting in an error when building docs. run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps --workspace --exclude=autonomi-cli - - name: Check local is not a default feature - shell: bash - run: if [[ ! $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].features.default[]? | select(. == "local")') ]]; then echo "local is not a default feature in any package."; else echo "local is a default feature in at least one package." && exit 1; fi - - name: Clean out the target directory run: cargo clean @@ -125,11 +121,11 @@ jobs: - name: Run autonomi tests timeout-minutes: 25 - run: cargo test --release --package autonomi --features full,local --lib + run: cargo test --release --package autonomi --lib - name: Run autonomi doc tests timeout-minutes: 25 - run: cargo test --release --package autonomi --features full,local --doc + run: cargo test --release --package autonomi --doc - name: Run bootstrap tests timeout-minutes: 25 @@ -148,19 +144,19 @@ jobs: # This is most likely due to the setup and cocurrency issues of the tests. # As the `record_store` is used in a single thread style, get the test passing executed # and passing standalone is enough. - - name: Run network tests (with encrypt-records) + - name: Run network tests timeout-minutes: 25 - run: cargo test --release --package ant-networking --features="open-metrics, encrypt-records" -- --skip can_store_after_restart + run: cargo test --release --package ant-networking --features="open-metrics" -- --skip can_store_after_restart - - name: Run network tests (with encrypt-records) + - name: Run network tests timeout-minutes: 5 - run: cargo test --release --package ant-networking --features="open-metrics, encrypt-records" can_store_after_restart + run: cargo test --release --package ant-networking --features="open-metrics" can_store_after_restart - - name: Run network tests (without encrypt-records) + - name: Run network tests timeout-minutes: 25 run: cargo test --release --package ant-networking --features="open-metrics" -- --skip can_store_after_restart - - name: Run network tests (without encrypt-records) + - name: Run network tests timeout-minutes: 5 run: cargo test --release --package ant-networking --features="open-metrics" can_store_after_restart @@ -172,15 +168,6 @@ jobs: timeout-minutes: 25 run: cargo test --release --package ant-logging - - name: Run register tests - timeout-minutes: 25 - run: cargo test --release --package ant-registers - env: - # this will speed up PR merge flows, while giving us a modicum - # of proptesting - # we do many more runs on the nightly run - PROPTEST_CASES: 50 - e2e: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" name: E2E tests @@ -202,7 +189,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -249,13 +236,13 @@ jobs: shell: pwsh - name: Get file cost - run: ./target/release/ant --log-output-dest=data-dir file cost "./resources" + run: ./target/release/ant --log-output-dest=data-dir --local file cost "./resources" env: ANT_LOG: "v" timeout-minutes: 15 - name: File upload - run: ./target/release/ant --log-output-dest=data-dir file upload "./resources" > ./upload_output 2>&1 + run: ./target/release/ant --log-output-dest=data-dir --local file upload "./resources" > ./upload_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 15 @@ -275,16 +262,16 @@ jobs: shell: pwsh - name: File Download - run: ./target/release/ant --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources + run: ./target/release/ant --log-output-dest=data-dir --local file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources env: ANT_LOG: "v" timeout-minutes: 5 - name: Generate register signing key - run: ./target/release/ant --log-output-dest=data-dir register generate-key + run: ./target/release/ant --log-output-dest=data-dir --local register generate-key - name: Create register (writeable by owner) - run: ./target/release/ant --log-output-dest=data-dir register create baobao 123 > ./register_create_output 2>&1 + run: ./target/release/ant --log-output-dest=data-dir --local register create baobao 123 > ./register_create_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 10 @@ -304,51 +291,51 @@ jobs: shell: pwsh - name: Get register - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} + run: ./target/release/ant --log-output-dest=data-dir --local register get ${{ env.REGISTER_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 5 - name: Edit register - run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.REGISTER_ADDRESS }} 456 + run: ./target/release/ant --log-output-dest=data-dir --local register edit ${{ env.REGISTER_ADDRESS }} 456 env: ANT_LOG: "v" timeout-minutes: 10 - name: Get register (after edit) - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} + run: ./target/release/ant --log-output-dest=data-dir --local register get ${{ env.REGISTER_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 5 - - name: Create Public Register (writeable by anyone) - run: ./target/release/ant --log-output-dest=data-dir register create bao 111 --public > ./register_public_create_output 2>&1 + - name: Create Register + run: ./target/release/ant --log-output-dest=data-dir --local register create bao 111 > ./register2_create_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 5 - - name: parse public register address (unix) + - name: parse register address (unix) if: matrix.os != 'windows-latest' run: | - PUBLIC_REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output) - echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" >> $GITHUB_ENV + REGISTER2_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register2_create_output) + echo "REGISTER2_ADDRESS=$REGISTER2_ADDRESS" >> $GITHUB_ENV shell: bash - - name: parse public register address (win) + - name: parse register address (win) if: matrix.os == 'windows-latest' run: | - $PUBLIC_REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output - echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + $REGISTER2_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register2_create_output + echo "REGISTER2_ADDRESS=$REGISTER2_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append shell: pwsh - - name: Get Public Register (current key is the owner) - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (current key is the owner) + run: ./target/release/ant --log-output-dest=data-dir --local register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 5 - - name: Edit Public Register (current key is the owner) - run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 222 + - name: Edit Register (current key is the owner) + run: ./target/release/ant --log-output-dest=data-dir --local register edit ${{ env.REGISTER2_ADDRESS }} 222 env: ANT_LOG: "v" timeout-minutes: 10 @@ -360,20 +347,14 @@ jobs: - name: Generate new register signing key run: ./target/release/ant --log-output-dest data-dir register generate-key - - name: Get Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (new signing key is not the owner) + run: ./target/release/ant --log-output-dest data-dir --local register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 2 - - name: Edit Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 333 - env: - ANT_LOG: "v" - timeout-minutes: 10 - - - name: Get Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (new signing key is not the owner) + run: ./target/release/ant --log-output-dest data-dir --local register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 2 @@ -385,25 +366,25 @@ jobs: timeout-minutes: 2 - name: file upload - run: ./target/release/ant --log-output-dest data-dir file upload random.txt + run: ./target/release/ant --log-output-dest data-dir --local file upload random.txt env: ANT_LOG: "v" timeout-minutes: 2 - name: create a local register - run: ./target/release/ant --log-output-dest data-dir register create sample_new_register 1234 + run: ./target/release/ant --log-output-dest data-dir --local register create sample_new_register 1234 env: ANT_LOG: "v" timeout-minutes: 2 - name: Estimate cost to create a vault - run: ./target/release/ant --log-output-dest data-dir vault cost + run: ./target/release/ant --log-output-dest data-dir --local vault cost env: ANT_LOG: "v" timeout-minutes: 2 - name: create a vault with existing user data as above - run: ./target/release/ant --log-output-dest data-dir vault create + run: ./target/release/ant --log-output-dest data-dir --local vault create env: ANT_LOG: "v" timeout-minutes: 2 @@ -414,9 +395,9 @@ jobs: set -e for i in {1..50}; do dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none - ./target/release/ant --log-output-dest data-dir file upload random_file_$i.bin --public - ./target/release/ant --log-output-dest data-dir file upload random_file_$i.bin - ./target/release/ant --log-output-dest data-dir register create $i random_file_$i.bin + ./target/release/ant --log-output-dest data-dir --local file upload random_file_$i.bin --public + ./target/release/ant --log-output-dest data-dir --local file upload random_file_$i.bin + ./target/release/ant --log-output-dest data-dir --local register create $i random_file_$i.bin done env: ANT_LOG: "v" @@ -433,22 +414,22 @@ jobs: [System.IO.File]::WriteAllBytes($fileName, $byteArray) # Run autonomi commands - ./target/release/ant --log-output-dest data-dir file upload "random_file_$i.bin" --public - ./target/release/ant --log-output-dest data-dir file upload "random_file_$i.bin" - ./target/release/ant --log-output-dest data-dir register create $i "random_file_$i.bin" + ./target/release/ant --log-output-dest data-dir --local file upload "random_file_$i.bin" --public + ./target/release/ant --log-output-dest data-dir --local file upload "random_file_$i.bin" + ./target/release/ant --log-output-dest data-dir --local register create $i "random_file_$i.bin" } env: ANT_LOG: "v" timeout-minutes: 25 - name: sync the vault - run: ./target/release/ant --log-output-dest data-dir vault sync + run: ./target/release/ant --log-output-dest data-dir --local vault sync env: ANT_LOG: "v" timeout-minutes: 2 - name: load the vault from network - run: ./target/release/ant --log-output-dest data-dir vault load + run: ./target/release/ant --log-output-dest data-dir --local vault load env: ANT_LOG: "v" timeout-minutes: 2 @@ -460,36 +441,25 @@ jobs: set -e NUM_OF_PUBLIC_FILES="" NUM_OF_PRIVATE_FILES="" - NUM_OF_REGISTERS="" NUM_OF_PUBLIC_FILES_IN_VAULT="" NUM_OF_PRIVATE_FILES_IN_VAULT="" - NUM_OF_REGISTERS_IN_VAULT="" - ./target/release/ant --log-output-dest data-dir file list 2>&1 > file_list.txt - - ./target/release/ant register list | grep register > register_list.txt + ./target/release/ant --log-output-dest data-dir --local file list 2>&1 > file_list.txt NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` - NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` - # when obtaining registers we get random garbage, this is the only hack that works. - NUM_OF_REGISTERS_first=${NUM_OF_REGISTERS%%[ $'\n']*} - echo "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" - ./target/release/ant --log-output-dest data-dir vault load 2>&1 > vault_data.txt + ./target/release/ant --log-output-dest data-dir --local vault load 2>&1 > vault_data.txt NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` - NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'` echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" - echo "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" - rm -rf file_list.txt register_list.txt vault_data.txt + rm -rf file_list.txt vault_data.txt python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: local data and vault in network dont match, Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: local data and vault in network dont match, Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT - python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: local data and vault in network dont match, Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS_first $NUM_OF_REGISTERS_IN_VAULT echo "vault synced successfully!" env: ANT_LOG: "v" @@ -500,9 +470,8 @@ jobs: shell: pwsh run: | $ErrorActionPreference = "Stop" - ./target/release/ant --log-output-dest data-dir file list > file_list.txt 2>&1 - ./target/release/ant register list > register_list.txt 2>&1 - ./target/release/ant --log-output-dest data-dir vault load > vault_data.txt 2>&1 + ./target/release/ant --log-output-dest data-dir --local file list > file_list.txt 2>&1 + ./target/release/ant --log-output-dest data-dir --local vault load > vault_data.txt 2>&1 env: ANT_LOG: "v" timeout-minutes: 15 @@ -535,26 +504,21 @@ jobs: print("NUM_OF_PUBLIC_FILES:", NUM_OF_PUBLIC_FILES) NUM_OF_PRIVATE_FILES = find_number_before_word("file_list.txt", "private") print("NUM_OF_PRIVATE_FILES:", NUM_OF_PRIVATE_FILES) - NUM_OF_REGISTERS_FILES = find_number_before_word("register_list.txt", "register") - print("NUM_OF_REGISTERS_FILES:", NUM_OF_REGISTERS_FILES) NUM_OF_PUBLIC_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "public") print("NUM_OF_PUBLIC_FILES_IN_VAULT:", NUM_OF_PUBLIC_FILES_IN_VAULT) NUM_OF_PRIVATE_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "private") print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT) - NUM_OF_REGISTERS_IN_VAULT = find_number_before_word("vault_data.txt", "register") - print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT) # Assertions assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: local data and vault in network dont match, Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: local data and vault in network dont match, Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" - assert NUM_OF_REGISTERS_FILES == NUM_OF_REGISTERS_IN_VAULT, f"Error: local data and vault in network dont match, Local registers: {NUM_OF_REGISTERS_FILES} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" print("Vault synced successfully!") env: ANT_LOG: "v" timeout-minutes: 2 - name: load an existing vault from the network - run: ./target/release/ant --log-output-dest=data-dir vault load + run: ./target/release/ant --log-output-dest data-dir --local vault load env: ANT_LOG: "v" timeout-minutes: 2 @@ -572,12 +536,12 @@ jobs: # 1 GB python3 -c "with open('random_1GB.bin', 'wb') as f: f.write(bytearray([0xff] * 1000 * 1024 * 1024))" - ./target/release/ant --log-output-dest=data-dir file list - time ./target/release/ant --log-output-dest=data-dir file upload random_1MB.bin - time ./target/release/ant --log-output-dest=data-dir file upload random_10MB.bin - time ./target/release/ant --log-output-dest=data-dir file upload random_100MB.bin - time ./target/release/ant --log-output-dest=data-dir file upload random_1GB.bin - ./target/release/ant --log-output-dest=data-dir vault sync + ./target/release/ant --log-output-dest=data-dir --local file list + time ./target/release/ant --log-output-dest=data-dir --local file upload random_1MB.bin + time ./target/release/ant --log-output-dest=data-dir --local file upload random_10MB.bin + time ./target/release/ant --log-output-dest=data-dir --local file upload random_100MB.bin + time ./target/release/ant --log-output-dest=data-dir --local file upload random_1GB.bin + ./target/release/ant --log-output-dest=data-dir --local vault sync rm -rf random*.bin rm -rf ${{ matrix.ant_path }}/autonomi env: @@ -592,138 +556,6 @@ jobs: log_file_prefix: safe_test_logs_e2e platform: ${{ matrix.os }} - # transaction_test: - # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" - # name: transaction tests against network - # runs-on: ${{ matrix.os }} - # strategy: - # matrix: - # os: [ubuntu-latest, windows-latest, macos-latest] - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Rust - # uses: dtolnay/rust-toolchain@stable - - # - uses: Swatinem/rust-cache@v2 - - # - name: Build binaries - # run: cargo build --release --features=local --bin antnode - # timeout-minutes: 30 - - # - name: Build faucet binary - # run: cargo build --release --bin faucet --features="local,gifting" - # timeout-minutes: 30 - - # - name: Start a local network - # uses: maidsafe/ant-local-testnet-action@main - # with: - # action: start - # interval: 2000 - # node-path: target/release/antnode - # faucet-path: target/release/faucet - # platform: ${{ matrix.os }} - # build: true - - # - name: Check ANT_PEERS was set - # shell: bash - # run: | - # if [[ -z "$ANT_PEERS" ]]; then - # echo "The ANT_PEERS variable has not been set" - # exit 1 - # else - # echo "ANT_PEERS has been set to $ANT_PEERS" - # fi - - # - name: execute the sequential transfers tests - # run: cargo test --release -p ant-node --features="local" --test sequential_transfers -- --nocapture --test-threads=1 - # env: - # ANT_LOG: "all" - # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - # timeout-minutes: 25 - - # - name: execute the storage payment tests - # run: cargo test --release -p ant-node --features="local" --test storage_payments -- --nocapture --test-threads=1 - # env: - # ANT_LOG: "all" - # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - # timeout-minutes: 25 - - # - name: Stop the local network and upload logs - # if: always() - # uses: maidsafe/ant-local-testnet-action@main - # with: - # action: stop - # log_file_prefix: safe_test_logs_transaction - # platform: ${{ matrix.os }} - - # # runs with increased node count - # transaction_simulation: - # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" - # name: transaction simulation - # runs-on: ${{ matrix.os }} - # strategy: - # matrix: - # os: [ ubuntu-latest, windows-latest, macos-latest ] - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Rust - # uses: dtolnay/rust-toolchain@stable - - # - uses: Swatinem/rust-cache@v2 - - # - name: Build binaries - # run: cargo build --release --features=local --bin antnode - # timeout-minutes: 30 - - # - name: Build faucet binary - # run: cargo build --release --bin faucet --features="local,gifting" - # timeout-minutes: 30 - - # - name: Build testing executable - # run: cargo test --release -p ant-node --features=local --test transaction_simulation --no-run - # env: - # # only set the target dir for windows to bypass the linker issue. - # # happens if we build the node manager via testnet action - # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - # timeout-minutes: 30 - - # - name: Start a local network - # uses: maidsafe/ant-local-testnet-action@main - # with: - # action: start - # interval: 2000 - # node-count: 50 - # node-path: target/release/antnode - # faucet-path: target/release/faucet - # platform: ${{ matrix.os }} - # build: true - - # - name: Check ANT_PEERS was set - # shell: bash - # run: | - # if [[ -z "$ANT_PEERS" ]]; then - # echo "The ANT_PEERS variable has not been set" - # exit 1 - # else - # echo "ANT_PEERS has been set to $ANT_PEERS" - # fi - - # - name: execute the transaction simulation - # run: cargo test --release -p ant-node --features="local" --test transaction_simulation -- --nocapture - # env: - # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - # timeout-minutes: 25 - - # - name: Stop the local network and upload logs - # if: always() - # uses: maidsafe/ant-local-testnet-action@main - # with: - # action: stop - # log_file_prefix: safe_test_logs_transaction_simulation - # platform: ${{ matrix.os }} - # token_distribution_test: # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" # name: token distribution test @@ -740,15 +572,15 @@ jobs: # - uses: Swatinem/rust-cache@v2 # - name: Build binaries - # run: cargo build --release --features=local,distribution --bin antnode + # run: cargo build --release --features=distribution --bin antnode # timeout-minutes: 35 # - name: Build faucet binary - # run: cargo build --release --features=local,distribution,gifting --bin faucet + # run: cargo build --release --features=distribution,gifting --bin faucet # timeout-minutes: 35 # - name: Build testing executable - # run: cargo test --release --features=local,distribution --no-run + # run: cargo test --release --features=distribution --no-run # env: # # only set the target dir for windows to bypass the linker issue. # # happens if we build the node manager via testnet action @@ -776,7 +608,7 @@ jobs: # fi # - name: execute token_distribution tests - # run: cargo test --release --features=local,distribution token_distribution -- --nocapture --test-threads=1 + # run: cargo test --release --features=distribution token_distribution -- --nocapture --test-threads=1 # env: # ANT_LOG: "all" # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} @@ -814,11 +646,11 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build binaries - run: cargo build --release --features local --bin antnode + run: cargo build --release --bin antnode timeout-minutes: 30 - name: Build churn tests - run: cargo test --release -p ant-node --features=local --test data_with_churn --no-run + run: cargo test --release -p ant-node --test data_with_churn --no-run env: # only set the target dir for windows to bypass the linker issue. # happens if we build the node manager via testnet action @@ -849,7 +681,7 @@ jobs: fi - name: Chunks data integrity during nodes churn - run: cargo test --release -p ant-node --features=local --test data_with_churn -- --nocapture + run: cargo test --release -p ant-node --test data_with_churn -- --nocapture env: TEST_DURATION_MINS: 5 TEST_TOTAL_CHURN_CYCLES: 15 @@ -961,11 +793,11 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build binaries - run: cargo build --release --features local --bin antnode + run: cargo build --release --bin antnode timeout-minutes: 30 - name: Build data location and routing table tests - run: cargo test --release -p ant-node --features=local --test verify_data_location --test verify_routing_table --no-run + run: cargo test --release -p ant-node --test verify_data_location --test verify_routing_table --no-run env: # only set the target dir for windows to bypass the linker issue. # happens if we build the node manager via testnet action @@ -995,14 +827,8 @@ jobs: echo "EVM_NETWORK has been set to $EVM_NETWORK" fi - - name: Verify the routing tables of the nodes - run: cargo test --release -p ant-node --features "local" --test verify_routing_table -- --nocapture - env: - CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - timeout-minutes: 5 - - name: Verify the location of the data on the network - run: cargo test --release -p ant-node --features "local" --test verify_data_location -- --nocapture + run: cargo test --release -p ant-node --test verify_data_location -- --nocapture env: CHURN_COUNT: 6 ANT_LOG: "all" @@ -1010,7 +836,7 @@ jobs: timeout-minutes: 25 - name: Verify the routing tables of the nodes - run: cargo test --release -p ant-node --features "local" --test verify_routing_table -- --nocapture + run: cargo test --release -p ant-node --test verify_routing_table -- --nocapture env: CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} timeout-minutes: 5 @@ -1284,7 +1110,7 @@ jobs: ls -l - name: Build binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -1327,7 +1153,7 @@ jobs: shell: bash - name: File upload - run: ./target/release/ant --log-output-dest data-dir file upload "./test_data_1.tar.gz" > ./upload_output 2>&1 + run: ./target/release/ant --log-output-dest data-dir --local file upload "./test_data_1.tar.gz" > ./upload_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 15 @@ -1344,7 +1170,7 @@ jobs: shell: bash - name: File Download - run: ./target/release/ant --log-output-dest data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources > ./download_output 2>&1 + run: ./target/release/ant --log-output-dest data-dir --local file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources > ./download_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 5 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8b4cc22cce..8b8eb0d401 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ jobs: continue-on-error: true - name: Build binaries - run: cargo build --release --features local --bin antnode --bin ant + run: cargo build --release --bin antnode --bin ant timeout-minutes: 30 - name: Start a local network @@ -152,34 +152,34 @@ jobs: ANT_LOG: "v" timeout-minutes: 5 - - name: Create Public Register (writeable by anyone) - run: ./target/release/ant --log-output-dest=data-dir register create bao 111 --public > ./register_public_create_output 2>&1 + - name: Create Register + run: ./target/release/ant --log-output-dest=data-dir register create bao 111 > ./register2_create_output 2>&1 env: ANT_LOG: "v" timeout-minutes: 5 - - name: parse public register address (unix) + - name: parse register address (unix) if: matrix.os != 'windows-latest' run: | - PUBLIC_REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output) - echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" >> $GITHUB_ENV + REGISTER2_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register2_create_output) + echo "REGISTER2_ADDRESS=$REGISTER2_ADDRESS" >> $GITHUB_ENV shell: bash - - name: parse public register address (win) + - name: parse register address (win) if: matrix.os == 'windows-latest' run: | - $PUBLIC_REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output - echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + $REGISTER2_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register2_create_output + echo "REGISTER2_ADDRESS=$REGISTER2_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append shell: pwsh - - name: Get Public Register (current key is the owner) - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (current key is the owner) + run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 5 - - name: Edit Public Register (current key is the owner) - run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 222 + - name: Edit Register (current key is the owner) + run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.REGISTER2_ADDRESS }} 222 env: ANT_LOG: "v" timeout-minutes: 10 @@ -191,20 +191,14 @@ jobs: - name: Generate new register signing key run: ./target/release/ant --log-output-dest=data-dir register generate-key - - name: Get Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (new signing key is not the owner) + run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 2 - - name: Edit Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 333 - env: - ANT_LOG: "v" - timeout-minutes: 10 - - - name: Get Public Register (new signing key is not the owner) - run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} + - name: Get Register (new signing key is not the owner) + run: ./target/release/ant --log-output-dest=data-dir register get ${{ env.REGISTER2_ADDRESS }} env: ANT_LOG: "v" timeout-minutes: 2 @@ -246,7 +240,7 @@ jobs: - name: Run autonomi tests timeout-minutes: 25 - run: cargo test --release --package autonomi --lib --features="full,fs" + run: cargo test --release --package autonomi --lib --features="full" - name: Run bootstrap tests timeout-minutes: 25 @@ -262,7 +256,7 @@ jobs: - name: Run network tests timeout-minutes: 25 - run: cargo test --release --package ant-networking --features="open-metrics, encrypt-records" + run: cargo test --release --package ant-networking --features="open-metrics" - name: Run protocol tests timeout-minutes: 25 @@ -272,12 +266,6 @@ jobs: timeout-minutes: 25 run: cargo test --release --package ant-logging - - name: Run register tests - timeout-minutes: 50 - run: cargo test --release --package ant-registers - env: - PROPTEST_CASES: 512 - - name: post notification to slack on failure if: ${{ failure() }} uses: bryannice/gitactions-slack-notification@2.0.0 @@ -311,11 +299,11 @@ jobs: continue-on-error: true - name: Build binaries - run: cargo build --release --features local --bin antnode + run: cargo build --release --bin antnode timeout-minutes: 30 - name: Build churn tests - run: cargo test --release -p ant-node --features=local --test data_with_churn --no-run + run: cargo test --release -p ant-node --test data_with_churn --no-run env: # only set the target dir for windows to bypass the linker issue. # happens if we build the node manager via testnet action @@ -332,7 +320,7 @@ jobs: build: true - name: Chunks data integrity during nodes churn (during 10min) (in theory) - run: cargo test --release -p ant-node --features=local --test data_with_churn -- --nocapture + run: cargo test --release -p ant-node --test data_with_churn -- --nocapture env: TEST_DURATION_MINS: 60 TEST_CHURN_CYCLES: 6 @@ -464,11 +452,11 @@ jobs: continue-on-error: true - name: Build binaries - run: cargo build --release --features local --bin antnode + run: cargo build --release --bin antnode timeout-minutes: 30 - name: Build data location and routing table tests - run: cargo test --release -p ant-node --features=local --test verify_data_location --test verify_routing_table --no-run + run: cargo test --release -p ant-node --test verify_data_location --test verify_routing_table --no-run env: # only set the target dir for windows to bypass the linker issue. # happens if we build the node manager via testnet action @@ -485,20 +473,20 @@ jobs: build: true - name: Verify the Routing table of the nodes - run: cargo test --release -p ant-node --features=local --test verify_routing_table -- --nocapture + run: cargo test --release -p ant-node --test verify_routing_table -- --nocapture env: CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} timeout-minutes: 5 - name: Verify the location of the data on the network - run: cargo test --release -p ant-node --features=local --test verify_data_location -- --nocapture + run: cargo test --release -p ant-node --test verify_data_location -- --nocapture env: ANT_LOG: "all" CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} timeout-minutes: 90 - name: Verify the routing tables of the nodes - run: cargo test --release -p ant-node --features=local --test verify_routing_table -- --nocapture + run: cargo test --release -p ant-node --test verify_routing_table -- --nocapture env: CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} timeout-minutes: 5 diff --git a/.github/workflows/nightly_wan.yml b/.github/workflows/nightly_wan.yml index 144fe88040..0b5ce3a7cc 100644 --- a/.github/workflows/nightly_wan.yml +++ b/.github/workflows/nightly_wan.yml @@ -147,99 +147,6 @@ jobs: SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" SLACK_TITLE: "Nightly E2E Test Run Failed" - # transaction_test: - # name: Spend tests against network - # runs-on: ${{ matrix.os }} - # strategy: - # matrix: - # os: [ubuntu-latest] - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Rust - # uses: dtolnay/rust-toolchain@stable - - # - uses: Swatinem/rust-cache@v2 - # continue-on-error: true - - # - name: setup testnet-deploy - # uses: maidsafe/sn-testnet-control-action/init-testnet-deploy@main - # with: - # ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }} - # aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }} - # aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }} - # aws-region: eu-west-2 - # do-token: ${{ secrets.SN_TESTNET_DO_PAT }} - # ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }} - - # - name: launch ${{ env.NETWORK_NAME }} - # uses: maidsafe/sn-testnet-control-action/launch-network@main - # with: - # ansible-forks: ${{ env.ANSIBLE_FORKS }} - # beta-encryption-key: ${{ env.DEFAULT_PAYMENT_FORWARD_SK }} - # environment-type: development - # faucet-version: ${{ env.FAUCET_VERSION }} - # log-format: json - # network-name: ${{ env.NETWORK_NAME }} - # network-contacts-file-name: ${{ env.NETWORK_CONTACTS_FILE_NAME }} - # provider: digital-ocean - # safe-network-branch: main - # safe-network-user: maidsafe - - # - name: Check env variables - # shell: bash - # run: | - # echo "Peer is $ANT_PEERS" - # echo "Deployment inventory is $SN_INVENTORY" - - # - name: execute the sequential transfers test - # run: cargo test --release -p ant-node --test sequential_transfers -- --nocapture --test-threads=1 - # env: - # ANT_LOG: "all" - # timeout-minutes: 45 - - # - name: execute the storage payment tests - # run: cargo test --release -p ant-node --test storage_payments -- --nocapture --test-threads=1 - # env: - # ANT_LOG: "all" - # timeout-minutes: 45 - - # - name: Small wait to allow reward receipt - # run: sleep 30 - # timeout-minutes: 1 - - # - name: Fetch network logs - # uses: ermineJose/sn-testnet-control-action/fetch-logs@feat-add_fetch-logs-action - # with: - # re-attempts: 3 - # rust-log: debug - # provider: digital-ocean - # testnet-name: ${{ env.NETWORK_NAME }} - - # - name: Upload local logs - # if: always() - # uses: actions/upload-artifact@v4 - # with: - # name: local_logs_NightlySpendTest - # path: | - # ~/.local/share/autonomi/node/*/logs/*.log* - # ~/.local/share/autonomi/*/*/*.log* - # ~/.local/share/autonomi/autonomi/logs/*/*.log* - - # - name: destroy network - # uses: maidsafe/sn-testnet-control-action/destroy-network@main - # with: - # network-name: ${{ env.NETWORK_NAME }} - # provider: digital-ocean - - # - name: post notification to slack on failure - # if: ${{ failure() }} - # uses: bryannice/gitactions-slack-notification@2.0.0 - # env: - # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - # SLACK_TITLE: "Nightly Spend Test Run Failed" - # churn: # name: Network churning tests # runs-on: ${{ matrix.os }} diff --git a/.github/workflows/node_man_tests.yml b/.github/workflows/node_man_tests.yml index b3de7a8f7c..9787e95b46 100644 --- a/.github/workflows/node_man_tests.yml +++ b/.github/workflows/node_man_tests.yml @@ -25,7 +25,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: cargo cache registry, index and build - uses: actions/cache@v4.1.2 + uses: actions/cache@v4.2.0 with: path: | ~/.cargo/registry diff --git a/.github/workflows/python-publish-client.yml b/.github/workflows/python-publish-client.yml index 5714ec7c22..12fa3e198f 100644 --- a/.github/workflows/python-publish-client.yml +++ b/.github/workflows/python-publish-client.yml @@ -3,8 +3,9 @@ name: Build and Publish Python Client Package on: push: tags: - - 'xxx' + - 'autonomi-v*' +# Add top-level permissions block permissions: id-token: write contents: read @@ -12,6 +13,7 @@ permissions: jobs: macos: runs-on: macos-latest + # Add permissions to job permissions: id-token: write contents: read @@ -24,40 +26,23 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" - EOL - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --compatibility manylinux2014 + args: --release --out dist -i python${{ matrix.python-version }} sccache: 'true' working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true windows: runs-on: windows-latest + # Add permissions to job permissions: id-token: write contents: read @@ -71,35 +56,18 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.target }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - shell: cmd - run: | - if not exist "autonomi\python\autonomi_client" mkdir autonomi\python\autonomi_client - echo from .autonomi_client import * > autonomi\python\autonomi_client\__init__.py - echo __version__ = "${{ github.ref_name }}" >> autonomi\python\autonomi_client\__init__.py - name: Build wheels uses: PyO3/maturin-action@v1 with: - args: --release --out dist --find-interpreter --compatibility manylinux2014 + args: --release --out dist sccache: 'true' working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-windows-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true linux: runs-on: ubuntu-latest @@ -108,131 +76,187 @@ jobs: contents: read strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] target: [x86_64] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - target: x86_64-unknown-linux-gnu - - name: Install dependencies - run: | - python -m pip install --user cffi - python -m pip install --user patchelf - rustup component add rustfmt - - name: Create Python module structure - run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" - EOL + architecture: x64 - name: Build wheels uses: PyO3/maturin-action@v1 + env: + PYTHON_VERSION: ${{ matrix.python-version }} with: target: ${{ matrix.target }} - manylinux: "2014" - args: --release --out dist --find-interpreter - sccache: 'true' - working-directory: ./autonomi + manylinux: auto before-script-linux: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env + rustup default stable rustup component add rustfmt + args: --release --out dist -i python${{ matrix.python-version }} + sccache: 'true' + working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-linux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 + + musllinux: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + matrix: + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - armv7-unknown-linux-musleabihf + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Build wheels + uses: PyO3/maturin-action@v1 env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true + PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} + PYO3_CROSS: "1" + with: + target: ${{ matrix.target }} + manylinux: musllinux_1_2 + args: --release --out dist -i python${{ matrix.python-version }} + sccache: 'true' + working-directory: ./autonomi + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl + if-no-files-found: error sdist: runs-on: ubuntu-latest + # Add permissions to job permissions: id-token: write contents: read steps: - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure + - name: Prepare standalone package run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" + # Create build directory structure + mkdir -p build/autonomi + cp -r autonomi/* build/autonomi/ + + # First, copy all workspace members + for dir in ant-* test-utils evmlib; do + if [ -d "$dir" ]; then + echo "Copying $dir to build directory" + cp -r "$dir" "build/$dir" + fi + done + + # Create a new workspace Cargo.toml in the build directory + cat > build/Cargo.toml << EOL + [workspace] + resolver = "2" + members = [ + "ant-bootstrap", + "ant-build-info", + "ant-cli", + "ant-evm", + "ant-logging", + "ant-metrics", + "ant-networking", + "ant-node", + "ant-node-manager", + "ant-node-rpc-client", + "ant-protocol", + "ant-service-management", + "ant-token-supplies", + "autonomi", + "evmlib", + "test-utils" + ] + + [workspace.lints.rust] + arithmetic_overflow = "forbid" + mutable_transmutes = "forbid" + no_mangle_const_items = "forbid" + trivial_casts = "warn" + trivial_numeric_casts = "warn" + unsafe_code = "warn" + unknown_crate_types = "forbid" + unused_extern_crates = "warn" + unused_import_braces = "warn" + + [workspace.lints.clippy] + clone_on_ref_ptr = "warn" + unicode_not_nfc = "warn" + uninlined_format_args = "warn" + unused_async = "warn" + unwrap_used = "warn" + + [profile.dev] + debug = 0 + strip = "debuginfo" + + [workspace.metadata.release] + pre-release-commit-message = "chore(release): release commit, tags, deps and changelog updates" + publish = false + push = false + tag = false + + [workspace.dependencies] + backtrace = "=0.3.71" EOL + + # Update all dependency paths to be absolute + find build -name "Cargo.toml" -exec sed -i "s|path = \"\.\./|path = \"/home/runner/work/autonomi/autonomi/build/|g" {} \; + + # Display directory structure for debugging + echo "Contents of build directory:" + ls -la build/ + echo "Contents of workspace Cargo.toml:" + cat build/Cargo.toml - name: Build sdist uses: PyO3/maturin-action@v1 with: command: sdist args: --out dist - working-directory: ./autonomi + working-directory: build/autonomi - name: Upload sdist uses: actions/upload-artifact@v4 with: - name: sdist - path: autonomi/dist/*.tar.gz + name: wheels + path: build/autonomi/dist/*.tar.gz if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true release: name: Release runs-on: ubuntu-latest - needs: [macos, windows, linux, sdist] + needs: [macos, windows, linux, musllinux, sdist] + # Keep existing permissions permissions: id-token: write contents: read steps: - - name: Create dist directory - run: mkdir -p dist - - # Download all artifacts at once - - name: Download all artifacts - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: + name: wheels path: dist - - - name: Prepare dist directory - run: | - find dist -type f -name "*.whl" -exec mv {} dist/ \; - find dist -type f -name "*.tar.gz" -exec mv {} dist/ \; - rm -rf dist/*/ - echo "Final dist directory contents:" - ls -la dist/ - - - name: Check if version exists - run: | - VERSION="${{ github.ref_name }}" - VERSION="${VERSION#v}" # Remove 'v' prefix if present - if pip index versions autonomi-client | grep -q "${VERSION}"; then - echo "Version ${VERSION} already exists on PyPI" - exit 1 - fi - + merge-multiple: true + - name: Display structure of downloaded files + run: ls -R dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ verbose: true - print-hash: true + print-hash: true \ No newline at end of file diff --git a/.github/workflows/python-publish-node.yml b/.github/workflows/python-publish-node.yml index e369bd2296..cf2aa8bc93 100644 --- a/.github/workflows/python-publish-node.yml +++ b/.github/workflows/python-publish-node.yml @@ -3,8 +3,9 @@ name: Build and Publish Python Node Package on: push: tags: - - 'xxx' + - 'ant-node-v*' +# Add top-level permissions block permissions: id-token: write contents: read @@ -12,6 +13,7 @@ permissions: jobs: macos: runs-on: macos-latest + # Add permissions to job permissions: id-token: write contents: read @@ -24,40 +26,23 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL - from ._antnode import * - __version__ = "${{ github.ref_name }}" - EOL - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist + args: --release --out dist -i python${{ matrix.python-version }} sccache: 'true' - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true windows: runs-on: windows-latest + # Add permissions to job permissions: id-token: write contents: read @@ -71,35 +56,18 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.target }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - shell: cmd - run: | - if not exist "ant_node\python\antnode" mkdir ant_node\python\antnode - echo from ._antnode import * > ant_node\python\antnode\__init__.py - echo __version__ = "${{ github.ref_name }}" >> ant_node\python\antnode\__init__.py - name: Build wheels uses: PyO3/maturin-action@v1 with: args: --release --out dist sccache: 'true' - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-windows-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true linux: runs-on: ubuntu-latest @@ -108,55 +76,69 @@ jobs: contents: read strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] target: [x86_64] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - target: x86_64-unknown-linux-gnu - - name: Install dependencies - run: | - python -m pip install --user cffi - python -m pip install --user patchelf - rustup component add rustfmt - - name: Create Python module structure - run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL - from ._antnode import * - __version__ = "${{ github.ref_name }}" - EOL + architecture: x64 - name: Build wheels uses: PyO3/maturin-action@v1 + env: + PYTHON_VERSION: ${{ matrix.python-version }} with: target: ${{ matrix.target }} manylinux: auto - args: --release --out dist - sccache: 'true' - working-directory: ./ant_node before-script-linux: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env + rustup default stable rustup component add rustfmt + args: --release --out dist -i python${{ matrix.python-version }} + sccache: false + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-linux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 + + musllinux: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + matrix: + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - armv7-unknown-linux-musleabihf + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Build wheels + uses: PyO3/maturin-action@v1 env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true + PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} + PYO3_CROSS: "1" + with: + target: ${{ matrix.target }} + manylinux: musllinux_1_2 + args: --release --out dist -i python${{ matrix.python-version }} + sccache: false + working-directory: ./ant-node + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl + if-no-files-found: error sdist: runs-on: ubuntu-latest @@ -171,8 +153,8 @@ jobs: components: rustfmt - name: Create Python module structure run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL + mkdir -p ant-node/python/antnode + cat > ant-node/python/antnode/__init__.py << EOL from ._antnode import * __version__ = "${{ github.ref_name }}" EOL @@ -181,12 +163,12 @@ jobs: with: command: sdist args: --out dist - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload sdist uses: actions/upload-artifact@v4 with: name: sdist - path: ant_node/dist/*.tar.gz + path: ant-node/dist/*.tar.gz if-no-files-found: error retention-days: 1 compression-level: 9 @@ -199,7 +181,7 @@ jobs: release: name: Release runs-on: ubuntu-latest - needs: [macos, windows, linux, sdist] + needs: [macos, windows, linux, musllinux, sdist] permissions: id-token: write contents: read @@ -226,4 +208,4 @@ jobs: with: packages-dir: dist/ verbose: true - print-hash: true + print-hash: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83065ae861..e2a583099c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -242,40 +242,40 @@ jobs: SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" SLACK_TITLE: "Release Failed" - publish-crates: - if: ${{ github.repository_owner == 'maidsafe' && github.ref == 'refs/heads/stable' }} - needs: [ build, s3-release ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: "0" - token: ${{ secrets.AUTONOMI_PAT }} - - uses: dtolnay/rust-toolchain@stable - - # Required for the creation of tags - - shell: bash - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - - - uses: cargo-bins/cargo-binstall@main - - shell: bash - run: cargo binstall --no-confirm release-plz - - - name: publish crates - shell: bash - run: | - cargo login "${{ secrets.CRATES_IO_TOKEN }}" - # The use of 'awk' suppresses the annoying instrumentation output that makes the log - # difficult to read. - release-plz release --git-token ${{ secrets.AUTONOMI_PAT }} | \ - awk '{ if (!/^\s*in release with input/ && !/^\s{4}/) print }' - - - name: post notification to slack on failure - if: ${{ failure() }} - uses: bryannice/gitactions-slack-notification@2.0.0 - env: - SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} - SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" - SLACK_TITLE: "Release Failed" + # publish-crates: + # if: ${{ github.repository_owner == 'maidsafe' && github.ref == 'refs/heads/stable' }} + # needs: [ build, s3-release ] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: "0" + # token: ${{ secrets.AUTONOMI_PAT }} + # - uses: dtolnay/rust-toolchain@stable + # + # # Required for the creation of tags + # - shell: bash + # run: | + # git config --local user.email "action@github.com" + # git config --local user.name "GitHub Action" + # + # - uses: cargo-bins/cargo-binstall@main + # - shell: bash + # run: cargo binstall --no-confirm release-plz + # + # - name: publish crates + # shell: bash + # run: | + # cargo login "${{ secrets.CRATES_IO_TOKEN }}" + # # The use of 'awk' suppresses the annoying instrumentation output that makes the log + # # difficult to read. + # release-plz release --git-token ${{ secrets.AUTONOMI_PAT }} | \ + # awk '{ if (!/^\s*in release with input/ && !/^\s{4}/) print }' + # + # - name: post notification to slack on failure + # if: ${{ failure() }} + # uses: bryannice/gitactions-slack-notification@2.0.0 + # env: + # SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + # SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + # SLACK_TITLE: "Release Failed" diff --git a/.gitignore b/.gitignore index d0e9a0da11..0a2abcfd3a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,11 @@ uv.lock *.swp /vendor/ + + +# Node.js +node_modules/ + +# MkDocs +site/ +/evmlib/tests/private.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1e4e3f91..081aed11e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *When editing this file, please respect a line length of 100.* +## 2025-02-11 + +### Network + +#### Changed + +- Removed encrypt data compile time flag (now always on). +- Refactor of data types. +- Removed the default trait for `QuotingMetrics` and it is now initialized with the correct values + everywhere. +- Compile UPnP support by default; will still require `--upnp` when launching the node to activate. +- Removed the old flawed `Register` native data type. +- Creating `DataTypes` as the sole place to show network natively supported data types. And use it to + replace existing `RecordKind`. +- Rename `RecordType` to `ValidationType`. +- Remove MDNS. For local nodes will bootstrap via the peer cache mechanism. +- Upgrade `libp2p` to `0.55.0` and use some small configuration changes it makes available. + +#### Added + +- `GraphEntry` data native type as a generic graph for building collections. +- `Pointer` data native type that points to other data on the network. +- Relay client events to the metrics endpoint. +- Relay reservation score to the metrics endpoint. This measures the health of a relay server that + we are connected to, by tracking all the recent connections that were routed through that server. +- Allow override QUIC max stream window with `ANT_MAX_STREAM_DATA`. +- Added an easy way to spawn nodes or an entire network from code, with `ant_node::spawn::node_spawner::NodeSpawner` and `ant_node::spawn::network_spawner::NetworkSpawner`. +- Added a `data_type` verification when receiving records with proof of payment. +- Added extra logging around payment verification. +- Make `QuotingMetrics` support data type variant pricing. +- Avoid free upload via replication. + +#### Fixed + +- External Address Manager will not consider `IncomingConnectionError` that originates from multiple + dial attempts as a serious issue. +- `MultiAddressNotSupported` error is not considered as a critical error if the error set contains + at least one different error. +- The record count metrics is now set as soon as a node is restarted. +- Push our Identify info if we make a new reservation with a relay server. This reduces the number + of `CircuitReqDenied` errors throughout the network. +- All connection errors are now more forgiving and does not result in a peer being evicted from the + routing table immediately. These errors are tracked and the action is taken only if we go over a + threshold. +- Only replicate fresh uploads to other payees. +- During quoting re-attempts, use non-blocking sleep instead. + +### Client + +#### Changed + +- Update python bindings and docs. Added initial NodeJS typescript integration. +- Updated test suit and added comprehensive documentation. +- Deprecate storing registers references in user data. +- Correctly report on chunks that were already uploaded to the network when syncing or re-uploading + the same data. +- Add version field to archive data structure for backwards compatibility. And add future + compatibility serialization into file metadata. +- Changed default EVM network to `Arbitrum One`. +- Removed the deprecated `Client::connect` function! Please use `Client::init` instead. +- Removed the old `Register` native data type, although the new `Register` high level type does the + same job but better. +- Removed the feature flags and the complexities around those, now everything is configurable at + runtime (no need to recompile). + +#### Added + +- NodeJS/Typescript bindings. +- 36 different configurations for publish Python bindings. +- Client examples. +- Added `evm_network` field to client config. +- Added a better retry strategy for getting market prices and sending transactions. This reduces the + frequency of RPC related upload errors significantly. +- Added a `data_type` verification when receiving quotes from nodes. +- Client API for all four data types: `Chunk`, `GraphEntry`, `Scratchpad`, `Pointer`. +- High level `Register` data type that works similarly to old registers but without the update limit + they had: now infinitely mutable. +- key derivation tooling + +#### Fixed + +- Rust optimization: Use parallelised chunk cloning in self encryption. +- Deterministically serialize archives. This leads to de-duplication and less payments when syncing + folders and files. +- Patched and refactored client Python bindings to reflect almost the whole Rust API. +- EVM network uses default if not supplied by ENV. +- Event receiver panic after completing client operations. + ## 2025-01-21 ### Client diff --git a/Cargo.lock b/Cargo.lock index 88be91b037..a3caeb7b84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,7 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.48" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0161082e0edd9013d23083465cc04b20e44b7a15646d36ba7b0cdb7cd6fe18f" +checksum = "4ab9d1367c6ffb90c93fb4a9a4989530aa85112438c6f73a734067255d348469" dependencies = [ "alloy-primitives", "num_enum", @@ -199,14 +199,14 @@ dependencies = [ "alloy-transport", "futures", "futures-util", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] name = "alloy-core" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c618bd382f0bc2ac26a7e4bfae01c9b015ca8f21b37ca40059ae35a7e62b3dc6" +checksum = "648275bb59110f88cc5fa9a176845e52a554ebfebac2d21220bcda8c9220f797" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -217,9 +217,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41056bde53ae10ffbbf11618efbe1e0290859e5eab0fe9ef82ebdb62f12a866f" +checksum = "bc9138f4f0912793642d453523c3116bd5d9e11de73b70177aa7cb3e94b98ad2" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -270,7 +270,7 @@ dependencies = [ "derive_more", "once_cell", "serde", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c357da577dfb56998d01f574d81ad7a1958d248740a7981b205d69d65a7da404" +checksum = "24acd2f5ba97c7a320e67217274bc81fe3c3174b8e6144ec875d9d54e760e278" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -307,7 +307,7 @@ dependencies = [ "alloy-sol-types", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.11", "tracing", ] @@ -333,7 +333,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] @@ -361,16 +361,16 @@ dependencies = [ "rand 0.8.5", "serde_json", "tempfile", - "thiserror 2.0.6", + "thiserror 2.0.11", "tracing", "url", ] [[package]] name = "alloy-primitives" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6259a506ab13e1d658796c31e6e39d2e2ee89243bcc505ddc613b35732e0a430" +checksum = "ec878088ec6283ce1e90d280316aadd3d6ce3de06ff63d68953c855e7e447e92" dependencies = [ "alloy-rlp", "bytes", @@ -379,8 +379,7 @@ dependencies = [ "derive_more", "foldhash", "hashbrown 0.15.2", - "hex-literal", - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "k256", "keccak-asm", @@ -390,7 +389,7 @@ dependencies = [ "ruint", "rustc-hash", "serde", - "sha3 0.10.8", + "sha3", "tiny-keccak", ] @@ -424,22 +423,22 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest 0.12.9", + "reqwest 0.12.12", "schnellru", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.11", "tokio", "tracing", "url", - "wasmtimer 0.4.1", + "wasmtimer", ] [[package]] name = "alloy-rlp" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f542548a609dca89fcd72b3b9f355928cf844d4363c5eed9c5273a3dd225e097" +checksum = "3d6c1d995bff8d011f7cd6c81820d51825e6e06d6db73914c1630ecf544d83d6" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -448,13 +447,13 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a833d97bf8a5f0f878daf2c8451fff7de7f9de38baa5a45d936ec718d81255a" +checksum = "a40e1ef334153322fd878d07e86af7a529bcb86b2439525920a88eba87bcf943" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -469,7 +468,7 @@ dependencies = [ "alloy-transport-http", "futures", "pin-project", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "serde_json", "tokio", @@ -477,7 +476,7 @@ dependencies = [ "tower 0.5.2", "tracing", "url", - "wasmtimer 0.4.1", + "wasmtimer", ] [[package]] @@ -555,9 +554,9 @@ dependencies = [ "alloy-primitives", "async-trait", "auto_impl", - "elliptic-curve 0.13.8", + "elliptic-curve", "k256", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] @@ -573,47 +572,47 @@ dependencies = [ "async-trait", "k256", "rand 0.8.5", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] name = "alloy-sol-macro" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d64f851d95619233f74b310f12bcf16e0cbc27ee3762b6115c14a84809280a" +checksum = "8d039d267aa5cbb7732fa6ce1fd9b5e9e29368f580f80ba9d7a8450c794de4b2" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "alloy-sol-macro-expander" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf7ed1574b699f48bf17caab4e6e54c6d12bc3c006ab33d58b1e227c1c3559f" +checksum = "620ae5eee30ee7216a38027dec34e0585c55099f827f92f50d11e3d2d3a4a954" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", "const-hex", "heck 0.5.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "syn-solidity", "tiny-keccak", ] [[package]] name = "alloy-sol-macro-input" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c02997ccef5f34f9c099277d4145f183b422938ed5322dc57a089fe9b9ad9ee" +checksum = "ad9f7d057e00f8c5994e4ff4492b76532c51ead39353aa2ed63f8c50c0f4d52e" dependencies = [ "alloy-json-abi", "const-hex", @@ -622,15 +621,15 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.90", + "syn 2.0.96", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce13ff37285b0870d0a0746992a4ae48efaf34b766ae4c2640fa15e5305f8e73" +checksum = "74e60b084fe1aef8acecda2743ff2d93c18ff3eb67a2d3b12f62582a1e66ef5e" dependencies = [ "serde", "winnow", @@ -638,9 +637,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1174cafd6c6d810711b4e00383037bdb458efc4fe3dbafafa16567e0320c54d8" +checksum = "c1382302752cd751efd275f4d6ef65877ddf61e0e6f5ac84ef4302b79a33a31a" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -661,13 +660,12 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.11", "tokio", "tower 0.5.2", "tracing", "url", - "wasm-bindgen-futures", - "wasmtimer 0.4.1", + "wasmtimer", ] [[package]] @@ -678,7 +676,7 @@ checksum = "2e02ffd5d93ffc51d72786e607c97de3b60736ca3e636ead0ec1f7dce68ea3fd" dependencies = [ "alloy-json-rpc", "alloy-transport", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde_json", "tower 0.5.2", "tracing", @@ -687,9 +685,9 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a5fd8fea044cc9a8c8a50bb6f28e31f0385d820f116c5b98f6f4e55d6e5590b" +checksum = "6917c79e837aa7b77b7a6dae9f89cbe15313ac161c4d3cfaf8909ef21f3d22d8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -763,17 +761,18 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] [[package]] name = "ant-bootstrap" -version = "0.1.4" +version = "0.1.5" dependencies = [ "ant-logging", "ant-protocol", @@ -783,7 +782,7 @@ dependencies = [ "dirs-next", "futures", "libp2p", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "serde_json", "tempfile", @@ -792,13 +791,12 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "wasmtimer 0.2.1", "wiremock", ] [[package]] name = "ant-build-info" -version = "0.1.23" +version = "0.1.24" dependencies = [ "chrono", "tracing", @@ -807,7 +805,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.3.6" +version = "0.3.7" dependencies = [ "ant-bootstrap", "ant-build-info", @@ -820,7 +818,7 @@ dependencies = [ "criterion", "dirs-next", "eyre", - "hex 0.4.3", + "hex", "indicatif", "prettytable", "rand 0.8.5", @@ -838,11 +836,11 @@ dependencies = [ [[package]] name = "ant-evm" -version = "0.1.8" +version = "0.1.9" dependencies = [ "custom_debug", "evmlib", - "hex 0.4.3", + "hex", "lazy_static", "libp2p", "rand 0.8.5", @@ -855,13 +853,12 @@ dependencies = [ "tiny-keccak", "tokio", "tracing", - "wasmtimer 0.2.1", "xor_name", ] [[package]] name = "ant-logging" -version = "0.2.45" +version = "0.2.46" dependencies = [ "chrono", "color-eyre", @@ -886,7 +883,7 @@ dependencies = [ [[package]] name = "ant-metrics" -version = "0.1.24" +version = "0.1.25" dependencies = [ "clap", "color-eyre", @@ -900,14 +897,13 @@ dependencies = [ [[package]] name = "ant-networking" -version = "0.3.4" +version = "0.3.5" dependencies = [ "aes-gcm-siv", "ant-bootstrap", "ant-build-info", "ant-evm", "ant-protocol", - "ant-registers", "assert_fs", "async-trait", "blsttc", @@ -916,12 +912,10 @@ dependencies = [ "exponential-backoff", "eyre", "futures", - "getrandom 0.2.15", - "hex 0.4.3", + "hex", "hkdf", - "hyper 0.14.31", + "hyper 0.14.32", "itertools 0.12.1", - "lazy_static", "libp2p", "libp2p-identity", "prometheus-client", @@ -929,26 +923,22 @@ dependencies = [ "rand 0.8.5", "rayon", "rmp-serde", - "self_encryption", "serde", - "sha2 0.10.8", + "sha2", "strum", "sysinfo", "thiserror 1.0.69", - "tiny-keccak", "tokio", "tracing", "uuid", "void", "walkdir", - "wasm-bindgen-futures", - "wasmtimer 0.2.1", "xor_name", ] [[package]] name = "ant-node" -version = "0.3.5" +version = "0.3.6" dependencies = [ "ant-bootstrap", "ant-build-info", @@ -956,7 +946,6 @@ dependencies = [ "ant-logging", "ant-networking", "ant-protocol", - "ant-registers", "ant-service-management", "assert_fs", "async-trait", @@ -974,7 +963,7 @@ dependencies = [ "eyre", "file-rotate", "futures", - "hex 0.4.3", + "hex", "itertools 0.12.1", "libp2p", "num-traits", @@ -983,7 +972,7 @@ dependencies = [ "pyo3", "rand 0.8.5", "rayon", - "reqwest 0.12.9", + "reqwest 0.12.12", "rmp-serde", "self_encryption", "serde", @@ -1005,7 +994,7 @@ dependencies = [ [[package]] name = "ant-node-manager" -version = "0.11.7" +version = "0.11.8" dependencies = [ "ant-bootstrap", "ant-build-info", @@ -1028,11 +1017,11 @@ dependencies = [ "libp2p-identity", "mockall 0.12.1", "nix 0.27.1", - "predicates 3.1.2", + "predicates 3.1.3", "prost 0.9.0", "rand 0.8.5", - "reqwest 0.12.9", - "semver 1.0.23", + "reqwest 0.12.12", + "semver 1.0.25", "serde", "serde_json", "service-manager", @@ -1048,7 +1037,7 @@ dependencies = [ [[package]] name = "ant-node-rpc-client" -version = "0.6.41" +version = "0.6.42" dependencies = [ "ant-build-info", "ant-logging", @@ -1059,7 +1048,7 @@ dependencies = [ "blsttc", "clap", "color-eyre", - "hex 0.4.3", + "hex", "libp2p", "libp2p-identity", "thiserror 1.0.69", @@ -1072,26 +1061,26 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "0.3.3" +version = "1.0.0" dependencies = [ "ant-build-info", "ant-evm", - "ant-registers", "blsttc", "bytes", "color-eyre", "crdts", "custom_debug", "dirs-next", - "exponential-backoff", - "hex 0.4.3", + "hex", "lazy_static", "libp2p", + "prometheus-client", "prost 0.9.0", + "rand 0.8.5", "rmp-serde", "serde", "serde_json", - "sha2 0.10.8", + "sha2", "thiserror 1.0.69", "tiny-keccak", "tonic 0.6.2", @@ -1100,23 +1089,6 @@ dependencies = [ "xor_name", ] -[[package]] -name = "ant-registers" -version = "0.4.7" -dependencies = [ - "blsttc", - "crdts", - "eyre", - "hex 0.4.3", - "proptest", - "rand 0.8.5", - "rmp-serde", - "serde", - "thiserror 1.0.69", - "tiny-keccak", - "xor_name", -] - [[package]] name = "ant-releases" version = "0.4.0" @@ -1128,8 +1100,8 @@ dependencies = [ "flate2", "lazy_static", "regex", - "reqwest 0.12.9", - "semver 1.0.23", + "reqwest 0.12.12", + "semver 1.0.25", "serde_json", "tar", "thiserror 1.0.69", @@ -1139,7 +1111,7 @@ dependencies = [ [[package]] name = "ant-service-management" -version = "0.4.7" +version = "0.4.8" dependencies = [ "ant-bootstrap", "ant-evm", @@ -1151,7 +1123,7 @@ dependencies = [ "libp2p-identity", "mockall 0.11.4", "prost 0.9.0", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "service-manager", @@ -1166,7 +1138,7 @@ dependencies = [ [[package]] name = "ant-token-supplies" -version = "0.1.62" +version = "0.1.63" dependencies = [ "dirs-next", "reqwest 0.11.27", @@ -1178,9 +1150,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arboard" @@ -1217,7 +1189,7 @@ dependencies = [ "ark-serialize 0.3.0", "ark-std 0.3.0", "derivative", - "num-bigint 0.4.6", + "num-bigint", "num-traits", "paste", "rustc_version 0.3.3", @@ -1237,7 +1209,7 @@ dependencies = [ "derivative", "digest 0.10.7", "itertools 0.10.5", - "num-bigint 0.4.6", + "num-bigint", "num-traits", "paste", "rustc_version 0.4.1", @@ -1270,7 +1242,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ - "num-bigint 0.4.6", + "num-bigint", "num-traits", "quote", "syn 1.0.109", @@ -1282,7 +1254,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ - "num-bigint 0.4.6", + "num-bigint", "num-traits", "proc-macro2", "quote", @@ -1307,7 +1279,7 @@ checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" dependencies = [ "ark-std 0.4.0", "digest 0.10.7", - "num-bigint 0.4.6", + "num-bigint", ] [[package]] @@ -1351,12 +1323,6 @@ dependencies = [ "serde", ] -[[package]] -name = "ascii" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" - [[package]] name = "asn1-rs" version = "0.6.2" @@ -1381,7 +1347,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -1393,7 +1359,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1416,7 +1382,7 @@ dependencies = [ "bstr", "doc-comment", "libc", - "predicates 3.1.2", + "predicates 3.1.3", "predicates-core", "predicates-tree", "wait-timeout", @@ -1431,7 +1397,7 @@ dependencies = [ "anstyle", "doc-comment", "globwalk", - "predicates 3.1.2", + "predicates 3.1.3", "predicates-core", "predicates-tree", "tempfile", @@ -1454,6 +1420,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-io" version = "2.4.0" @@ -1464,7 +1442,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.5.0", + "futures-lite 2.6.0", "parking", "polling", "rustix", @@ -1479,11 +1457,22 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "event-listener-strategy", "pin-project-lite", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -1503,18 +1492,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1530,11 +1519,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atomic-write-file" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e32862ecc63d580f4a5e1436a685f51e0629caeb7a7933e4f017d5e2099e13" +checksum = "aeb1e2c1d58618bea806ccca5bbe65dc4e868be16f69ff118a39049389687548" dependencies = [ "nix 0.29.0", "rand 0.8.5", @@ -1564,22 +1559,13 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" +checksum = "e12882f59de5360c748c4cbf569a042d5fb0eb515f7bea9c1f470b47f6ffbd73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", -] - -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.4.0", + "syn 2.0.96", ] [[package]] @@ -1590,7 +1576,7 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "autonomi" -version = "0.3.5" +version = "0.3.6" dependencies = [ "alloy", "ant-bootstrap", @@ -1598,41 +1584,31 @@ dependencies = [ "ant-logging", "ant-networking", "ant-protocol", - "ant-registers", "bip39", "blst", "blstrs 0.7.1", "blsttc", "bytes", - "console_error_panic_hook", "const-hex", - "evmlib", "eyre", "futures", - "hex 0.4.3", - "instant", - "js-sys", + "hex", "libp2p", "pyo3", + "pyo3-async-runtimes", "rand 0.8.5", + "rayon", "rmp-serde", "self_encryption", "serde", - "serde-wasm-bindgen", - "sha2 0.10.8", - "sn_bls_ckd", - "sn_curv", + "serial_test", + "sha2", "test-utils", "thiserror 1.0.69", - "tiny_http", "tokio", "tracing", "tracing-subscriber", - "tracing-web", "walkdir", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test", "xor_name", ] @@ -1649,7 +1625,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "itoa", "matchit", "memchr", @@ -1702,12 +1678,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base16ct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" - [[package]] name = "base16ct" version = "0.2.0" @@ -1770,18 +1740,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin-internals" @@ -1807,9 +1777,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" dependencies = [ "serde", ] @@ -1835,59 +1805,22 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding 0.1.5", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "block-padding 0.2.1", - "generic-array 0.14.7", -] - [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", + "generic-array", ] -[[package]] -name = "block-padding" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" - [[package]] name = "block-padding" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -1953,7 +1886,7 @@ dependencies = [ "blstrs 0.6.2", "ff 0.12.1", "group 0.12.1", - "hex 0.4.3", + "hex", "hex_fmt", "pairing 0.22.0", "rand 0.8.5", @@ -1996,9 +1929,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata 0.4.9", @@ -2007,9 +1940,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "byte-slice-cast" @@ -2017,17 +1950,11 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -2080,7 +2007,7 @@ dependencies = [ "blst", "cc", "glob", - "hex 0.4.3", + "hex", "libc", "once_cell", "serde", @@ -2112,7 +2039,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" dependencies = [ "camino", "cargo-platform", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "thiserror 1.0.69", @@ -2159,9 +2086,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -2219,12 +2146,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "chunked_transfer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" - [[package]] name = "ciborium" version = "0.2.2" @@ -2265,9 +2186,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -2285,9 +2206,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -2300,14 +2221,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2325,15 +2246,6 @@ dependencies = [ "error-code", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "clru" version = "0.6.2" @@ -2375,19 +2287,19 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] name = "compact_str" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" dependencies = [ "castaway", "cfg-if", @@ -2428,25 +2340,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ - "encode_unicode 0.3.6", - "lazy_static", + "encode_unicode", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", -] - -[[package]] -name = "console_error_panic_hook" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" -dependencies = [ - "cfg-if", - "wasm-bindgen", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", ] [[package]] @@ -2457,7 +2359,7 @@ checksum = "4b0485bab839b018a8f1723fc5391819fea5f8f0f32288ef8a735fd096b6160c" dependencies = [ "cfg-if", "cpufeatures", - "hex 0.4.3", + "hex", "proptest", "serde", ] @@ -2554,9 +2456,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -2618,18 +2520,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2646,9 +2548,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" @@ -2656,7 +2558,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "crossterm_winapi", "futures-core", "libc", @@ -2674,7 +2576,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "crossterm_winapi", "mio 1.0.3", "parking_lot", @@ -2695,21 +2597,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-bigint" -version = "0.4.9" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array 0.14.7", - "rand_core 0.6.4", - "subtle", - "zeroize", -] +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-bigint" @@ -2717,7 +2607,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.7", + "generic-array", "rand_core 0.6.4", "subtle", "zeroize", @@ -2729,21 +2619,11 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array", "rand_core 0.6.4", "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" -dependencies = [ - "generic-array 0.14.7", - "subtle", -] - [[package]] name = "csv" version = "1.3.1" @@ -2774,19 +2654,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "curve25519-dalek" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" -dependencies = [ - "byteorder", - "digest 0.9.0", - "rand_core 0.5.1", - "subtle", - "zeroize", -] - [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -2811,7 +2678,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2832,7 +2699,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -2857,7 +2724,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2868,7 +2735,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2887,15 +2754,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "data-encoding-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +checksum = "5b16d9d0d88a5273d830dac8b78ceb217ffc9b1d5404e5597a3542515329405b" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -2903,12 +2770,12 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +checksum = "1145d32e826a7748b69ee8fc62d3e6355ff7f1051df53141e7048162fc90481b" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] @@ -2930,16 +2797,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.9" @@ -2959,7 +2816,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint 0.4.6", + "num-bigint", "num-traits", "rusticata-macros", ] @@ -3013,7 +2870,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "unicode-xid", ] @@ -3029,22 +2886,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -3053,7 +2901,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "const-oid", "crypto-common", "subtle", @@ -3129,7 +2977,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -3165,30 +3013,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "ecdsa" -version = "0.14.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" -dependencies = [ - "der 0.6.1", - "elliptic-curve 0.12.3", - "rfc6979 0.3.1", - "signature 1.6.4", -] - [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.7.9", + "der", "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", + "elliptic-curve", + "rfc6979", + "signature", + "spki", ] [[package]] @@ -3197,8 +3033,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "pkcs8", + "signature", ] [[package]] @@ -3207,11 +3043,11 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ - "curve25519-dalek 4.1.3", + "curve25519-dalek", "ed25519", "rand_core 0.6.4", "serde", - "sha2 0.10.8", + "sha2", "subtle", "zeroize", ] @@ -3222,51 +3058,25 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" -[[package]] -name = "elliptic-curve" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" -dependencies = [ - "base16ct 0.1.1", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest 0.10.7", - "ff 0.12.1", - "generic-array 0.14.7", - "group 0.12.1", - "pkcs8 0.9.0", - "rand_core 0.6.4", - "sec1 0.3.0", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", + "base16ct", + "crypto-bigint", "digest 0.10.7", "ff 0.13.0", - "generic-array 0.14.7", + "generic-array", "group 0.13.0", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "sec1 0.7.3", + "sec1", "subtle", "zeroize", ] -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encode_unicode" version = "1.0.0" @@ -3291,7 +3101,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -3334,9 +3144,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -3349,13 +3159,13 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "pin-project-lite", ] [[package]] name = "evm-testnet" -version = "0.1.8" +version = "0.1.9" dependencies = [ "ant-evm", "clap", @@ -3366,17 +3176,17 @@ dependencies = [ [[package]] name = "evmlib" -version = "0.1.8" +version = "0.1.9" dependencies = [ "alloy", "dirs-next", - "getrandom 0.2.15", "rand 0.8.5", "serde", "serde_with", "thiserror 1.0.69", "tokio", "tracing", + "tracing-subscriber", ] [[package]] @@ -3409,12 +3219,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "faster-hex" version = "0.9.0" @@ -3478,32 +3282,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ff-zeroize" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02169a2e8515aa316ce516eaaf6318a76617839fbf904073284bc2576b029ee" -dependencies = [ - "byteorder", - "ff_derive-zeroize", - "rand_core 0.5.1", - "zeroize", -] - -[[package]] -name = "ff_derive-zeroize" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b24d4059bc0d0a0bf26b740aa21af1f96a984f0ab7a21356d00b32475388b53a" -dependencies = [ - "num-bigint 0.2.6", - "num-integer", - "num-traits", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "fiat-crypto" version = "0.2.9" @@ -3557,7 +3335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide 0.8.3", ] [[package]] @@ -3569,6 +3347,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3577,9 +3364,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "foreign-types" @@ -3599,7 +3386,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -3629,12 +3416,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "funty" version = "2.0.0" @@ -3717,9 +3498,9 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ "futures-core", "pin-project-lite", @@ -3733,7 +3514,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -3743,7 +3524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", ] @@ -3759,26 +3540,11 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" -[[package]] -name = "futures-ticker" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9763058047f713632a52e916cc7f6a4b3fc6e9fc1ff8c5b1dc49e5a89041682e" -dependencies = [ - "futures", - "futures-timer", - "instant", -] - [[package]] name = "futures-timer" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" -dependencies = [ - "gloo-timers", - "send_wrapper 0.4.0", -] [[package]] name = "futures-util" @@ -3805,12 +3571,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" [[package]] -name = "generic-array" -version = "0.12.4" +name = "generator" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" dependencies = [ - "typenum", + "cfg-if", + "libc", + "log", + "rustversion", + "windows 0.58.0", ] [[package]] @@ -3858,13 +3628,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "ghash" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ - "opaque-debug 0.3.1", + "opaque-debug", "polyval", ] @@ -3932,20 +3714,20 @@ dependencies = [ [[package]] name = "gix-bitmap" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48b897b4bbc881aea994b4a5bbb340a04979d7be9089791304e04a9fbc66b53" +checksum = "b1db9765c69502650da68f0804e3dc2b5f8ccc6a2d104ca6c85bc40700d37540" dependencies = [ - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] name = "gix-chunk" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ffbeb3a5c0b8b84c3fe4133a6f8c82fa962f4caefe8d0762eced025d3eb4f7" +checksum = "0b1f1d8764958699dc764e3f727cef280ff4d1bd92c107bbf8acd85b30c1bd6f" dependencies = [ - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] @@ -3985,15 +3767,15 @@ dependencies = [ [[package]] name = "gix-config-value" -version = "0.14.10" +version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aaeef5d98390a3bcf9dbc6440b520b793d1bf3ed99317dc407b02be995b28e" +checksum = "11365144ef93082f3403471dbaa94cfe4b5e72743bdb9560719a251d439f4cee" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "bstr", "gix-path", "libc", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] @@ -4072,7 +3854,7 @@ version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74908b4bbc0a0a40852737e5d7889f676f081e340d5451a16e5b4c50d592f111" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "bstr", "gix-features", "gix-path", @@ -4105,7 +3887,7 @@ version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a9a44eb55bd84bb48f8a44980e951968ced21e171b22d115d1cdcef82a7d73f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "bstr", "filetime", "fnv", @@ -4146,7 +3928,7 @@ checksum = "999ce923619f88194171a67fb3e6d613653b8d4d6078b529b15a765da0edcc17" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -4208,26 +3990,26 @@ dependencies = [ [[package]] name = "gix-path" -version = "0.10.13" +version = "0.10.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc292ef1a51e340aeb0e720800338c805975724c1dfbd243185452efd8645b7" +checksum = "c40f12bb65a8299be0cfb90fe718e3be236b7a94b434877012980863a883a99f" dependencies = [ "bstr", "gix-trace", "home", "once_cell", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] name = "gix-quote" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a1e282216ec2ab2816cd57e6ed88f8009e634aec47562883c05ac8a7009a63" +checksum = "e49357fccdb0c85c0d3a3292a9f6db32d9b3535959b5471bb9624908f4a066c6" dependencies = [ "bstr", "gix-utils", - "thiserror 2.0.6", + "thiserror 2.0.11", ] [[package]] @@ -4299,11 +4081,11 @@ dependencies = [ [[package]] name = "gix-sec" -version = "0.10.10" +version = "0.10.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8b876ef997a955397809a2ec398d6a45b7a55b4918f2446344330f778d14fd6" +checksum = "d84dae13271f4313f8d60a166bf27e54c968c7c33e2ffd31c48cafe5da649875" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "gix-path", "libc", "windows-sys 0.52.0", @@ -4326,9 +4108,9 @@ dependencies = [ [[package]] name = "gix-trace" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04bdde120c29f1fc23a24d3e115aeeea3d60d8e65bab92cc5f9d90d9302eb952" +checksum = "7c396a2036920c69695f760a65e7f2677267ccf483f25046977d87e4cb2665f7" [[package]] name = "gix-traverse" @@ -4336,7 +4118,7 @@ version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e499a18c511e71cf4a20413b743b9f5bcf64b3d9e81e9c3c6cd399eae55a8840" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "gix-commitgraph", "gix-date", "gix-hash", @@ -4363,9 +4145,9 @@ dependencies = [ [[package]] name = "gix-utils" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba427e3e9599508ed98a6ddf8ed05493db114564e338e41f6a996d2e4790335f" +checksum = "ff08f24e03ac8916c478c8419d7d3c33393da9bb41fa4c24455d5406aeefd35f" dependencies = [ "fastrand 2.3.0", "unicode-normalization", @@ -4383,9 +4165,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "globset" @@ -4406,23 +4188,11 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "ignore", "walkdir", ] -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "group" version = "0.12.1" @@ -4432,7 +4202,7 @@ dependencies = [ "ff 0.12.1", "rand 0.8.5", "rand_core 0.6.4", - "rand_xorshift 0.3.0", + "rand_xorshift", "subtle", ] @@ -4445,7 +4215,7 @@ dependencies = [ "ff 0.13.0", "rand 0.8.5", "rand_core 0.6.4", - "rand_xorshift 0.3.0", + "rand_xorshift", "subtle", ] @@ -4461,7 +4231,26 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.7.0", + "indexmap 2.7.1", + "slab", + "tokio", + "tokio-util 0.7.13", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.2.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util 0.7.13", @@ -4521,6 +4310,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "headers" version = "0.3.9" @@ -4554,12 +4352,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -4587,12 +4379,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" -[[package]] -name = "hex" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" - [[package]] name = "hex" version = "0.4.3" @@ -4608,12 +4394,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212ab92002354b4819390025006c897e8140934349e8635c9b077f47b4dcbd20" -[[package]] -name = "hex-literal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" - [[package]] name = "hex_fmt" version = "0.3.0" @@ -4622,10 +4402,11 @@ checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] name = "hickory-proto" -version = "0.24.2" +version = "0.25.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" +checksum = "d063c0692ee669aa6d261988aa19ca5510f1cc40e4f211024f50c888499a35d7" dependencies = [ + "async-recursion", "async-trait", "cfg-if", "data-encoding", @@ -4638,7 +4419,7 @@ dependencies = [ "once_cell", "rand 0.8.5", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.11", "tinyvec", "tokio", "tracing", @@ -4647,21 +4428,21 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.2" +version = "0.25.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" +checksum = "42bc352e4412fb657e795f79b4efcf2bd60b59ee5ca0187f3554194cd1107a27" dependencies = [ "cfg-if", "futures-util", "hickory-proto", "ipconfig", - "lru-cache", + "moka", "once_cell", "parking_lot", "rand 0.8.5", "resolv-conf", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -4672,17 +4453,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac", ] [[package]] @@ -4696,11 +4467,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4777,7 +4548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", - "async-channel", + "async-channel 1.9.0", "base64 0.13.1", "futures-lite 1.13.0", "http 0.2.12", @@ -4793,9 +4564,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -4821,15 +4592,15 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -4845,13 +4616,14 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.7", "http 1.2.0", "http-body 1.0.1", "httparse", @@ -4870,7 +4642,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -4878,15 +4650,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.6.0", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", "tokio", "tokio-rustls 0.26.1", @@ -4900,7 +4672,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.31", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -4917,7 +4689,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -5063,7 +4835,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -5128,16 +4900,18 @@ dependencies = [ [[package]] name = "igd-next" -version = "0.14.3" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4" +checksum = "76b0d7d4541def58a37bf8efc559683f21edce7c82f0d866c93ac21f7e098f93" dependencies = [ "async-trait", "attohttpc", "bytes", "futures", - "http 0.2.12", - "hyper 0.14.31", + "http 1.2.0", + "http-body-util", + "hyper 1.6.0", + "hyper-util", "log", "rand 0.8.5", "tokio", @@ -5191,7 +4965,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -5206,16 +4980,16 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg 1.4.0", + "autocfg", "hashbrown 0.12.3", "serde", ] [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -5224,9 +4998,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.9" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", "number_prefix", @@ -5254,22 +5028,21 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "block-padding 0.3.3", - "generic-array 0.14.7", + "block-padding", + "generic-array", ] [[package]] name = "instability" -version = "0.3.3" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b829f37dead9dc39df40c2d3376c179fdfd2ac771f53f55d3c30dc096a3c0c6e" +checksum = "0bf9fed6d91cfb734e7476a06bde8300a1b94e217e1b523b6f0cd1a01998c71d" dependencies = [ "darling", "indoc", - "pretty_assertions", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -5279,9 +5052,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", ] [[package]] @@ -5298,19 +5068,19 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi 0.4.0", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5369,9 +5139,9 @@ checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -5395,10 +5165,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", + "ecdsa", + "elliptic-curve", "once_cell", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -5422,27 +5192,21 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.168" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" - -[[package]] -name = "libm" -version = "0.2.11" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libp2p" -version = "0.54.1" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbe80f9c7e00526cd6b838075b9c171919404a4732cb2fa8ece0a093223bfc4" +checksum = "b72dc443ddd0254cb49a794ed6b6728400ee446a0f7ab4a07d0209ee98de20e9" dependencies = [ "bytes", "either", @@ -5468,35 +5232,32 @@ dependencies = [ "libp2p-tcp", "libp2p-upnp", "libp2p-websocket", - "libp2p-websocket-websys", "libp2p-yamux", "multiaddr", "pin-project", "rw-stream-sink", - "thiserror 1.0.69", + "thiserror 2.0.11", ] [[package]] name = "libp2p-allow-block-list" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1027ccf8d70320ed77e984f273bc8ce952f623762cb9bf2d126df73caef8041" +checksum = "38944b7cb981cc93f2f0fb411ff82d0e983bd226fbcc8d559639a3a73236568b" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-autonat" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a083675f189803d0682a2726131628e808144911dad076858bfbe30b13065499" +checksum = "e297bfc6cabb70c6180707f8fa05661b77ecb9cb67e8e8e1c469301358fa21d0" dependencies = [ "async-trait", "asynchronous-codec", - "bytes", "either", "futures", "futures-bounded", @@ -5509,29 +5270,27 @@ dependencies = [ "quick-protobuf-codec", "rand 0.8.5", "rand_core 0.6.4", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-connection-limits" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d003540ee8baef0d254f7b6bfd79bac3ddf774662ca0abf69186d517ef82ad8" +checksum = "efe9323175a17caa8a2ed4feaf8a548eeef5e0b72d03840a0eab4bcb0210ce1c" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-core" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61f26c83ed111104cd820fe9bc3aaabbac5f1652a1d213ed6e900b7918a1298" +checksum = "193c75710ba43f7504ad8f58a62ca0615b1d7e572cb0f1780bc607252c39e9ef" dependencies = [ "either", "fnv", @@ -5547,20 +5306,17 @@ dependencies = [ "quick-protobuf", "rand 0.8.5", "rw-stream-sink", - "serde", - "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "unsigned-varint 0.8.0", - "void", "web-time", ] [[package]] name = "libp2p-dns" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f37f30d5c7275db282ecd86e54f29dd2176bd3ac656f06abf43bedb21eb8bd" +checksum = "1b780a1150214155b0ed1cdf09fbd2e1b0442604f9146a431d1b21d23eef7bd7" dependencies = [ "async-trait", "futures", @@ -5574,10 +5330,11 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.47.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4e830fdf24ac8c444c12415903174d506e1e077fbe3875c404a78c5935a8543" +checksum = "d558548fa3b5a8e9b66392f785921e363c57c05dcadfda4db0d41ae82d313e4a" dependencies = [ + "async-channel 2.3.1", "asynchronous-codec", "base64 0.22.1", "byteorder", @@ -5585,8 +5342,9 @@ dependencies = [ "either", "fnv", "futures", - "futures-ticker", + "futures-timer", "getrandom 0.2.15", + "hashlink 0.9.1", "hex_fmt", "libp2p-core", "libp2p-identity", @@ -5597,18 +5355,16 @@ dependencies = [ "rand 0.8.5", "regex", "serde", - "sha2 0.10.8", - "smallvec", + "sha2", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-identify" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1711b004a273be4f30202778856368683bd9a83c4c7dcc8f848847606831a4e3" +checksum = "e8c06862544f02d05d62780ff590cc25a75f5c2b9df38ec7a370dcae8bb873cf" dependencies = [ "asynchronous-codec", "either", @@ -5618,13 +5374,11 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "lru", "quick-protobuf", "quick-protobuf-codec", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", - "void", ] [[package]] @@ -5640,7 +5394,7 @@ dependencies = [ "quick-protobuf", "rand 0.8.5", "serde", - "sha2 0.10.8", + "sha2", "thiserror 1.0.69", "tracing", "zeroize", @@ -5648,11 +5402,10 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.46.2" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced237d0bd84bbebb7c2cad4c073160dacb4fe40534963c32ed6d4c6bb7702a3" +checksum = "2bab0466a27ebe955bcbc27328fae5429c5b48c915fd6174931414149802ec23" dependencies = [ - "arrayvec", "asynchronous-codec", "bytes", "either", @@ -5667,22 +5420,20 @@ dependencies = [ "quick-protobuf-codec", "rand 0.8.5", "serde", - "sha2 0.10.8", + "sha2", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", - "uint", - "void", + "uint 0.10.0", "web-time", ] [[package]] name = "libp2p-mdns" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b8546b6644032565eb29046b42744aee1e9f261ed99671b2c93fb140dba417" +checksum = "11d0ba095e1175d797540e16b62e7576846b883cb5046d4159086837b36846cc" dependencies = [ - "data-encoding", "futures", "hickory-proto", "if-watch", @@ -5694,14 +5445,13 @@ dependencies = [ "socket2", "tokio", "tracing", - "void", ] [[package]] name = "libp2p-metrics" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ebafa94a717c8442d8db8d3ae5d1c6a15e30f2d347e0cd31d057ca72e42566" +checksum = "2ce58c64292e87af624fcb86465e7dd8342e46a388d71e8fec0ab37ee789630a" dependencies = [ "futures", "libp2p-core", @@ -5717,13 +5467,12 @@ dependencies = [ [[package]] name = "libp2p-noise" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b137cb1ae86ee39f8e5d6245a296518912014eaa87427d24e6ff58cfc1b28c" +checksum = "afcc133e0f3cea07acde6eb8a9665cb11b600bd61110b010593a0210b8153b16" dependencies = [ "asynchronous-codec", "bytes", - "curve25519-dalek 4.1.3", "futures", "libp2p-core", "libp2p-identity", @@ -5732,10 +5481,9 @@ dependencies = [ "once_cell", "quick-protobuf", "rand 0.8.5", - "sha2 0.10.8", "snow", "static_assertions", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "x25519-dalek", "zeroize", @@ -5743,33 +5491,31 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46352ac5cd040c70e88e7ff8257a2ae2f891a4076abad2c439584a31c15fd24e" +checksum = "41432a159b00424a0abaa2c80d786cddff81055ac24aa127e0cf375f7858d880" dependencies = [ - "bytes", "futures", "futures-timer", "if-watch", "libp2p-core", "libp2p-identity", "libp2p-tls", - "parking_lot", "quinn", "rand 0.8.5", "ring 0.17.8", - "rustls 0.23.20", + "rustls 0.23.21", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.11", "tokio", "tracing", ] [[package]] name = "libp2p-relay" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10df23d7f5b5adcc129f4a69d6fbd05209e356ccf9e8f4eb10b2692b79c77247" +checksum = "08a41e346681395877118c270cf993f90d57d045fbf0913ca2f07b59ec6062e4" dependencies = [ "asynchronous-codec", "bytes", @@ -5784,23 +5530,21 @@ dependencies = [ "quick-protobuf-codec", "rand 0.8.5", "static_assertions", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-request-response" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1356c9e376a94a75ae830c42cdaea3d4fe1290ba409a22c809033d1b7dcab0a6" +checksum = "548fe44a80ff275d400f1b26b090d441d83ef73efabbeb6415f4ce37e5aed865" dependencies = [ "async-trait", "cbor4ii", "futures", "futures-bounded", - "futures-timer", "libp2p-core", "libp2p-identity", "libp2p-swarm", @@ -5808,21 +5552,18 @@ dependencies = [ "serde", "smallvec", "tracing", - "void", - "web-time", ] [[package]] name = "libp2p-swarm" -version = "0.45.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7dd6741793d2c1fb2088f67f82cf07261f25272ebe3c0b0c311e0c6b50e851a" +checksum = "803399b4b6f68adb85e63ab573ac568154b193e9a640f03e0f2890eabbcb37f8" dependencies = [ "either", "fnv", "futures", "futures-timer", - "getrandom 0.2.15", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", @@ -5833,8 +5574,6 @@ dependencies = [ "smallvec", "tokio", "tracing", - "void", - "wasm-bindgen-futures", "web-time", ] @@ -5847,21 +5586,20 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "libp2p-tcp" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad964f312c59dcfcac840acd8c555de8403e295d39edf96f5240048b5fcaa314" +checksum = "65346fb4d36035b23fec4e7be4c320436ba53537ce9b6be1d1db1f70c905cad0" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "libp2p-identity", "socket2", "tokio", "tracing", @@ -5869,9 +5607,9 @@ dependencies = [ [[package]] name = "libp2p-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b23dddc2b9c355f73c1e36eb0c3ae86f7dc964a3715f0731cfad352db4d847" +checksum = "dcaebc1069dea12c5b86a597eaaddae0317c2c2cb9ec99dc94f82fd340f5c78b" dependencies = [ "futures", "futures-rustls", @@ -5879,18 +5617,18 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.8", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-webpki 0.101.7", - "thiserror 1.0.69", + "thiserror 2.0.11", "x509-parser", "yasna", ] [[package]] name = "libp2p-upnp" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01bf2d1b772bd3abca049214a3304615e6a36fa6ffc742bdd1ba774486200b8f" +checksum = "d457b9ecceb66e7199f049926fad447f1f17f040e8d29d690c086b4cab8ed14a" dependencies = [ "futures", "futures-timer", @@ -5899,14 +5637,13 @@ dependencies = [ "libp2p-swarm", "tokio", "tracing", - "void", ] [[package]] name = "libp2p-websocket" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "888b2ff2e5d8dcef97283daab35ad1043d18952b65e05279eecbe02af4c6e347" +checksum = "2bf5d48a4d8fad8a49fbf23816a878cac25623549f415d74da8ef4327e6196a9" dependencies = [ "either", "futures", @@ -5917,40 +5654,22 @@ dependencies = [ "pin-project-lite", "rw-stream-sink", "soketto", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "url", "webpki-roots 0.25.4", ] -[[package]] -name = "libp2p-websocket-websys" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf9b429dd07be52cd82c4c484b1694df4209210a7db3b9ffb00c7606e230c8" -dependencies = [ - "bytes", - "futures", - "js-sys", - "libp2p-core", - "parking_lot", - "send_wrapper 0.6.0", - "thiserror 1.0.69", - "tracing", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "libp2p-yamux" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "788b61c80789dba9760d8c669a5bedb642c8267555c803fabd8396e4ca5c5882" +checksum = "f15df094914eb4af272acf9adaa9e287baa269943f32ea348ba29cfb9bfc60d8" dependencies = [ "either", "futures", "libp2p-core", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "yamux 0.12.1", "yamux 0.13.4", @@ -5962,22 +5681,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", "redox_syscall", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -5991,32 +5704,36 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ - "autocfg 1.4.0", + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] -name = "lru" -version = "0.12.5" +name = "loom" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ - "hashbrown 0.15.2", + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", ] [[package]] -name = "lru-cache" -version = "0.1.2" +name = "lru" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "linked-hash-map", + "hashbrown 0.15.2", ] [[package]] @@ -6061,16 +5778,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ - "autocfg 1.4.0", -] - -[[package]] -name = "merkle-cbt" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171d2f700835121c3b04ccf0880882987a050fd5c7ae88148abf537d33dd3a56" -dependencies = [ - "cfg-if", + "autocfg", ] [[package]] @@ -6089,16 +5797,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minicov" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" -dependencies = [ - "cc", - "walkdir", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -6116,9 +5814,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", "simd-adler32", @@ -6174,7 +5872,7 @@ dependencies = [ "fragile", "lazy_static", "mockall_derive 0.12.1", - "predicates 3.1.2", + "predicates 3.1.3", "predicates-tree", ] @@ -6199,7 +5897,26 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", +] + +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "loom", + "parking_lot", + "portable-atomic", + "rustc_version 0.4.1", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "uuid", ] [[package]] @@ -6283,7 +6000,7 @@ dependencies = [ [[package]] name = "nat-detection" -version = "0.2.15" +version = "0.2.16" dependencies = [ "ant-build-info", "ant-networking", @@ -6338,17 +6055,16 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b33524dc0968bfad349684447bfce6db937a9ac3332a1fe60c0c5a5ce63f21" +checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60" dependencies = [ "bytes", "futures", "log", "netlink-packet-core", "netlink-sys", - "thiserror 1.0.69", - "tokio", + "thiserror 2.0.11", ] [[package]] @@ -6381,7 +6097,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "libc", ] @@ -6392,7 +6108,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -6400,7 +6116,7 @@ dependencies = [ [[package]] name = "node-launchpad" -version = "0.5.3" +version = "0.5.4" dependencies = [ "ant-bootstrap", "ant-build-info", @@ -6432,7 +6148,7 @@ dependencies = [ "prometheus-parse", "ratatui", "regex", - "reqwest 0.12.9", + "reqwest 0.12.12", "serde", "serde_json", "signal-hook", @@ -6492,17 +6208,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.4.0", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -6511,7 +6216,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -6535,8 +6239,7 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "autocfg 1.4.0", - "libm", + "autocfg", ] [[package]] @@ -6566,7 +6269,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -6586,9 +6289,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "nybbles" -version = "0.2.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f06be0417d97f81fe4e5c86d7d01b392655a9cac9c19a848aa033e18937b23" +checksum = "8983bb634df7248924ee0c4c3a749609b5abcb082c28fffe3254b3eb3602b307" dependencies = [ "alloy-rlp", "const-hex", @@ -6619,7 +6322,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "block2", "libc", "objc2", @@ -6635,7 +6338,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "block2", "objc2", "objc2-foundation", @@ -6655,9 +6358,9 @@ dependencies = [ [[package]] name = "objc2-encode" -version = "4.0.3" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7891e71393cd1f227313c9379a26a584ff3d7e6e7159e988851f0934c993f0f8" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "objc2-foundation" @@ -6665,7 +6368,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "block2", "libc", "objc2", @@ -6677,7 +6380,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "block2", "objc2", "objc2-foundation", @@ -6689,7 +6392,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "block2", "objc2", "objc2-foundation", @@ -6726,12 +6429,6 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -6854,9 +6551,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.9.0" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ca711d8b83edbb00b44d504503cd247c9c0bd8b0fa2694f2a1a3d8165379ce" +checksum = "6e6520c8cc998c5741ee68ec1dc369fc47e5f0ea5320018ecf2a1ccd6328f48b" dependencies = [ "log", "serde", @@ -6875,17 +6572,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "p256" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" -dependencies = [ - "ecdsa 0.14.8", - "elliptic-curve 0.12.3", - "sha2 0.10.8", -] - [[package]] name = "pairing" version = "0.22.0" @@ -6904,21 +6590,6 @@ dependencies = [ "group 0.13.0", ] -[[package]] -name = "pairing-plus" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cda4f22e8e6720f3c254049960c8cc4f93cb82b5ade43bddd2622b5f39ea62" -dependencies = [ - "byteorder", - "digest 0.8.1", - "ff-zeroize", - "rand 0.4.6", - "rand_core 0.5.1", - "rand_xorshift 0.2.0", - "zeroize", -] - [[package]] name = "parity-scale-codec" version = "3.6.12" @@ -7004,9 +6675,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.7", - "hmac 0.12.1", + "hmac", "password-hash", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -7032,7 +6703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 2.0.11", "ucd-trie", ] @@ -7056,7 +6727,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -7067,7 +6738,7 @@ checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" dependencies = [ "once_cell", "pest", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -7077,34 +6748,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", ] [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -7112,24 +6783,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", -] - [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.9", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -7145,7 +6806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42cf17e9a1800f5f396bc67d193dc9411b59012a5876445ef450d449881e1016" dependencies = [ "base64 0.22.1", - "indexmap 2.7.0", + "indexmap 2.7.1", "quick-xml", "serde", "time", @@ -7181,15 +6842,15 @@ dependencies = [ [[package]] name = "png" -version = "0.17.15" +version = "0.17.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67582bd5b65bdff614270e2ea89a1cf15bef71245cc1e5f7ea126977144211d" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" dependencies = [ "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", - "miniz_oxide 0.8.0", + "miniz_oxide 0.8.3", ] [[package]] @@ -7214,7 +6875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ "cpufeatures", - "opaque-debug 0.3.1", + "opaque-debug", "universal-hash", ] @@ -7226,7 +6887,7 @@ checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", "cpufeatures", - "opaque-debug 0.3.1", + "opaque-debug", "universal-hash", ] @@ -7258,7 +6919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ "difflib", - "float-cmp", + "float-cmp 0.9.0", "itertools 0.10.5", "normalize-line-endings", "predicates-core", @@ -7267,13 +6928,13 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.2" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" dependencies = [ "anstyle", "difflib", - "float-cmp", + "float-cmp 0.10.0", "normalize-line-endings", "predicates-core", "regex", @@ -7281,15 +6942,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" [[package]] name = "predicates-tree" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" dependencies = [ "predicates-core", "termtree", @@ -7312,7 +6973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" dependencies = [ "csv", - "encode_unicode 1.0.0", + "encode_unicode", "is-terminal", "lazy_static", "term", @@ -7327,7 +6988,7 @@ checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", "impl-codec", - "uint", + "uint 0.9.5", ] [[package]] @@ -7358,14 +7019,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -7396,7 +7057,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -7413,18 +7074,18 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.6.0", + "bitflags 2.8.0", "lazy_static", "num-traits", "rand 0.8.5", "rand_chacha 0.3.1", - "rand_xorshift 0.3.0", + "rand_xorshift", "regex-syntax 0.8.5", "rusty-fork", "tempfile", @@ -7509,15 +7170,15 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -7525,11 +7186,24 @@ dependencies = [ "unindent", ] +[[package]] +name = "pyo3-async-runtimes" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "977dc837525cfd22919ba6a831413854beb7c99a256c03bf8624ad707e45810e" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -7537,9 +7211,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -7547,27 +7221,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -7630,9 +7304,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.20", + "rustls 0.23.21", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -7648,10 +7322,10 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -7659,9 +7333,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -7673,9 +7347,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -7686,38 +7360,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift 0.1.1", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -7728,7 +7370,7 @@ dependencies = [ "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc 0.2.0", + "rand_hc", ] [[package]] @@ -7743,16 +7385,6 @@ dependencies = [ "serde", ] -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", -] - [[package]] name = "rand_chacha" version = "0.2.2" @@ -7773,21 +7405,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -7806,15 +7423,6 @@ dependencies = [ "getrandom 0.2.15", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rand_hc" version = "0.2.0" @@ -7824,68 +7432,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_xorshift" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_xorshift" version = "0.3.0" @@ -7901,7 +7447,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cassowary", "compact_str", "crossterm 0.28.1", @@ -7949,22 +7495,13 @@ dependencies = [ "yasna", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -8033,10 +7570,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -8065,9 +7602,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64 0.22.1", "bytes", @@ -8076,8 +7613,8 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-util", "ipnet", "js-sys", @@ -8087,7 +7624,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -8096,6 +7633,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-rustls 0.26.1", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -8121,24 +7659,13 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" -dependencies = [ - "crypto-bigint 0.4.9", - "hmac 0.12.1", - "zeroize", -] - [[package]] name = "rfc6979" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac 0.12.1", + "hmac", "subtle", ] @@ -8211,7 +7738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.6.0", + "bitflags 2.8.0", "serde", "serde_derive", ] @@ -8266,7 +7793,7 @@ dependencies = [ "ark-ff 0.4.2", "bytes", "fastrlp", - "num-bigint 0.4.6", + "num-bigint", "num-traits", "parity-scale-codec", "primitive-types", @@ -8328,7 +7855,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.25", ] [[package]] @@ -8342,11 +7869,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -8380,9 +7907,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", "ring 0.17.8", @@ -8412,9 +7939,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] @@ -8442,9 +7969,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rusty-fork" @@ -8471,9 +7998,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -8484,11 +8011,20 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e1c91382686d21b5ac7959341fcb9780fa7c03773646995a87c950fa7be640" +dependencies = [ + "sdd", +] + [[package]] name = "schnellru" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9a8ef13a93c54d20580de1e5c413e624e53121d42fc7e2c11d10ef7f8b02367" +checksum = "356285bbf17bea63d9e52e96bd18f039672ac92b55b8cb997d6162a2a37d1649" dependencies = [ "ahash", "cfg-if", @@ -8528,18 +8064,10 @@ dependencies = [ ] [[package]] -name = "sec1" -version = "0.3.0" +name = "sdd" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" -dependencies = [ - "base16ct 0.1.1", - "der 0.6.1", - "generic-array 0.14.7", - "pkcs8 0.9.0", - "subtle", - "zeroize", -] +checksum = "478f121bb72bbf63c52c93011ea1791dca40140dfe13f8336c4c5ac952c33aa9" [[package]] name = "sec1" @@ -8547,34 +8075,14 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct 0.2.0", - "der 0.7.9", - "generic-array 0.14.7", - "pkcs8 0.10.2", + "base16ct", + "der", + "generic-array", + "pkcs8", "subtle", "zeroize", ] -[[package]] -name = "secp256k1" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d03ceae636d0fed5bae6a7f4f664354c5f4fcedf6eef053fef17e49f837d0a" -dependencies = [ - "rand 0.6.5", - "secp256k1-sys", - "serde", -] - -[[package]] -name = "secp256k1-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" -dependencies = [ - "cc", -] - [[package]] name = "self_encryption" version = "0.30.0" @@ -8586,7 +8094,7 @@ dependencies = [ "brotli", "bytes", "cbc", - "hex 0.4.3", + "hex", "itertools 0.10.5", "lazy_static", "num_cpus", @@ -8612,9 +8120,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -8628,63 +8136,31 @@ dependencies = [ "pest", ] -[[package]] -name = "send_wrapper" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" - -[[package]] -name = "send_wrapper" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" - [[package]] name = "serde" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] -[[package]] -name = "serde-wasm-bindgen" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" -dependencies = [ - "js-sys", - "serde", - "wasm-bindgen", -] - -[[package]] -name = "serde_bytes" -version = "0.11.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -8735,15 +8211,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ "base64 0.22.1", "chrono", - "hex 0.4.3", + "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_derive", "serde_json", @@ -8753,14 +8229,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -8769,13 +8245,38 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "service-manager" version = "0.7.1" @@ -8806,31 +8307,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.1", -] - [[package]] name = "sha2" version = "0.10.8" @@ -8842,18 +8318,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha3" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "keccak", - "opaque-debug 0.3.1", -] - [[package]] name = "sha3" version = "0.10.8" @@ -8920,16 +8384,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "2.2.0" @@ -8952,7 +8406,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg 1.4.0", + "autocfg", ] [[package]] @@ -8964,51 +8418,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sn_bls_ckd" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc1905b0d5c8c8dd4cfafa1b064645e8eb57c26ad93a491acbaa2dc59c3d8c2" -dependencies = [ - "hex 0.3.2", - "hkdf", - "sha2 0.10.8", - "sn_curv", -] - -[[package]] -name = "sn_curv" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b53f6b7e77c36e00b5469e77386c11d5a8d863300acb4bd373227894e3a117" -dependencies = [ - "curve25519-dalek 3.2.0", - "digest 0.9.0", - "ff-zeroize", - "generic-array 0.14.7", - "hex 0.4.3", - "hmac 0.11.0", - "lazy_static", - "merkle-cbt", - "num-bigint 0.4.6", - "num-integer", - "num-traits", - "p256", - "pairing-plus", - "rand 0.6.5", - "rand 0.7.3", - "secp256k1", - "serde", - "serde_bytes", - "serde_derive", - "sha2 0.8.2", - "sha2 0.9.9", - "sha3 0.9.1", - "thiserror 1.0.69", - "typenum", - "zeroize", -] - [[package]] name = "snow" version = "0.9.6" @@ -9018,11 +8427,11 @@ dependencies = [ "aes-gcm", "blake2", "chacha20poly1305", - "curve25519-dalek 4.1.3", + "curve25519-dalek", "rand_core 0.6.4", "ring 0.17.8", "rustc_version 0.4.1", - "sha2 0.10.8", + "sha2", "subtle", ] @@ -9063,16 +8472,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -9080,7 +8479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.9", + "der", ] [[package]] @@ -9097,9 +8496,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strip-ansi-escapes" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" dependencies = [ "vte", ] @@ -9129,7 +8528,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9151,9 +8550,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -9162,14 +8561,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219389c1ebe89f8333df8bdfb871f6631c552ff399c23cac02480b6088aad8f0" +checksum = "b84e4d83a0a6704561302b917a932484e1cae2d8c6354c64be8b7bac1c1fe057" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9195,7 +8594,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9230,7 +8629,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation", "system-configuration-sys 0.6.0", ] @@ -9255,6 +8654,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -9280,12 +8685,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand 2.3.0", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -9314,13 +8720,13 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-utils" -version = "0.4.15" +version = "0.4.16" dependencies = [ "bytes", "color-eyre", @@ -9343,11 +8749,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.11", ] [[package]] @@ -9358,18 +8764,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9454,19 +8860,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tiny_http" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d6ef4e10d23c1efb862eecad25c5054429a71958b4eeef85eb5e7170b477ca" -dependencies = [ - "ascii", - "chunked_transfer", - "log", - "time", - "url", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -9489,9 +8882,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -9504,9 +8897,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -9532,13 +8925,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9568,7 +8961,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.21", "tokio", ] @@ -9650,7 +9043,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -9669,10 +9062,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -9701,10 +9094,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -9759,6 +9152,7 @@ dependencies = [ "futures-util", "pin-project-lite", "sync_wrapper 1.0.2", + "tokio", "tower-layer", "tower-service", ] @@ -9807,7 +9201,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -9927,20 +9321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.90", -] - -[[package]] -name = "tracing-web" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e6a141feebd51f8d91ebfd785af50fca223c570b86852166caa3b141defe7c" -dependencies = [ - "js-sys", - "tracing-core", - "tracing-subscriber", - "wasm-bindgen", - "web-sys", + "syn 2.0.96", ] [[package]] @@ -9998,7 +9379,19 @@ checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ "byteorder", "crunchy", - "hex 0.4.3", + "hex", + "static_assertions", +] + +[[package]] +name = "uint" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" +dependencies = [ + "byteorder", + "crunchy", + "hex", "static_assertions", ] @@ -10010,9 +9403,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bom" @@ -10022,9 +9415,9 @@ checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-normalization" @@ -10170,18 +9563,18 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ "getrandom 0.2.15", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vergen" @@ -10212,22 +9605,11 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vte" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" -dependencies = [ - "utf8parse", - "vte_generate_state_changes", -] - -[[package]] -name = "vte_generate_state_changes" -version = "0.1.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" dependencies = [ - "proc-macro2", - "quote", + "memchr", ] [[package]] @@ -10275,7 +9657,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "mime", "mime_guess", @@ -10305,36 +9687,46 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -10345,9 +9737,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10355,61 +9747,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" - -[[package]] -name = "wasm-bindgen-test" -version = "0.3.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d44563646eb934577f2772656c7ad5e9c90fac78aa8013d776fcdaf24625d" -dependencies = [ - "js-sys", - "minicov", - "scoped-tls", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test-macro", -] - -[[package]] -name = "wasm-bindgen-test-macro" -version = "0.3.49" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54171416ce73aa0b9c377b51cc3cb542becee1cd678204812e8392e5b0e4a031" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", -] - -[[package]] -name = "wasmtimer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ed9d8b15c7fb594d72bfb4b5a276f3d2029333cd93a932f376f5937f6f80ee" -dependencies = [ - "futures", - "js-sys", - "parking_lot", - "pin-utils", - "serde", - "slab", - "wasm-bindgen", + "unicode-ident", ] [[package]] @@ -10428,9 +9783,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -10558,6 +9913,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -10577,6 +9942,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.2.0", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "windows-registry" version = "0.2.0" @@ -10766,9 +10166,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310" dependencies = [ "memchr", ] @@ -10802,7 +10202,7 @@ dependencies = [ "futures", "futures-timer", "http-types", - "hyper 0.14.31", + "hyper 0.14.32", "log", "once_cell", "regex", @@ -10811,6 +10211,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -10855,7 +10264,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ - "curve25519-dalek 4.1.3", + "curve25519-dalek", "rand_core 0.6.4", "serde", "zeroize", @@ -10880,9 +10289,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" dependencies = [ "libc", "linux-raw-sys", @@ -10891,9 +10300,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmltree" @@ -10910,7 +10319,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd9dddecfdbc7c17ae93da6d28a5a9c4f5564abe7b735d2530c7a159b6b55e8" dependencies = [ - "hex 0.4.3", + "hex", "rand 0.8.5", "rand_core 0.6.4", "serde", @@ -10926,7 +10335,7 @@ checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.8.4", ] [[package]] @@ -10995,7 +10404,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -11017,7 +10426,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -11037,7 +10446,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -11058,7 +10467,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -11080,7 +10489,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -11096,7 +10505,7 @@ dependencies = [ "crc32fast", "crossbeam-utils", "flate2", - "hmac 0.12.1", + "hmac", "pbkdf2", "sha1", "time", diff --git a/Cargo.toml b/Cargo.toml index 6840a1e40d..9b620b320b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ members = [ "ant-node-manager", "ant-node-rpc-client", "ant-protocol", - "ant-registers", "ant-service-management", "ant-token-supplies", "autonomi", @@ -50,3 +49,6 @@ pre-release-commit-message = "chore(release): release commit, tags, deps and cha publish = false push = false tag = false + +[workspace.dependencies] +backtrace = "=0.3.71" diff --git a/README.md b/README.md index c6a1f504f4..c7ea7932f3 100644 --- a/README.md +++ b/README.md @@ -63,9 +63,6 @@ More options about EVM Network below. The Autonomi network uses `quic` as the default transport protocol. -#### Building for wasm32 - -WASM support for the autonomi API is currently under active development. More docs coming soon. ### For the Technical @@ -77,8 +74,6 @@ WASM support for the autonomi API is currently under active development. More do networking layer, built atop libp2p which allows nodes and clients to communicate. - [Protocol](https://github.com/maidsafe/autonomi/blob/main/ant-protocol/README.md) The protocol used by the autonomi network. -- [Registers](https://github.com/maidsafe/autonomi/blob/main/ant-registers/README.md) The - registers crate, used for the Register CRDT data type on the network. - [Bootstrap](https://github.com/maidsafe/autonomi/blob/main/ant-bootstrap/README.md) The network bootstrap cache or: how the network layer discovers bootstrap peers. - [Build Info](https://github.com/maidsafe/autonomi/blob/main/ant-build-info/README.md) Small @@ -113,15 +108,15 @@ This creates a CSV file with the EVM network params in your data directory. `--rewards-address` _is the address where you will receive your node earnings on._ ```bash -cargo run --bin antctl --features local -- local run --build --clean --rewards-address +cargo run --bin antctl -- local run --build --clean --rewards-address ``` -The EVM Network parameters are loaded from the CSV file in your data directory automatically when the `local` feature flag is enabled (`--features=local`). +The EVM Network parameters are loaded from the CSV file in your data directory automatically when the `local` mode is enabled. ##### 4. Verify node status ```bash -cargo run --bin antctl --features local -- status +cargo run --bin antctl -- status ``` The Antctl `run` command starts the node processes. The `status` command should show twenty-five @@ -134,7 +129,7 @@ To upload a file or a directory, you need to set the `SECRET_KEY` environment va > When running a local network, you can use the `SECRET_KEY` printed by the `evm-testnet` command [step 2](#2-run-a-local-evm-node) as it has all the money. ```bash -SECRET_KEY= cargo run --bin ant --features local -- file upload +SECRET_KEY= cargo run --bin ant -- --local file upload ``` The output will print out the address at which the content was uploaded. @@ -142,120 +137,9 @@ The output will print out the address at which the content was uploaded. Now to download the files again: ```bash -cargo run --bin ant --features local -- file download -``` - -### Registers - -Registers are one of the network's data types. The workspace here has an example app demonstrating -their use by two users to exchange text messages in a crude chat application. - -In the first terminal, using the registers example, Alice creates a register: - -``` -cargo run --example registers --features=local -- --user alice --reg-nickname myregister -``` - -Alice can now write a message to the register and see anything written by anyone else. For example -she might enter the text "Hello, who's there?" which is written to the register and then shown as -the "Latest value", in her terminal: - -``` -Register address: "50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d" -Register owned by: PublicKey(0cf4..08a5) -Register permissions: Permissions { anyone_can_write: true, writers: {PublicKey(0cf4..08a5)} } - -Current total number of items in Register: 0 -Latest value (more than one if concurrent writes were made): --------------- --------------- - -Enter a blank line to receive updates, or some text to be written. -Hello, who's there? -Writing msg (offline) to Register: 'Hello, who's there?' -Syncing with SAFE in 2s... -synced! - -Current total number of items in Register: 1 -Latest value (more than one if concurrent writes were made): --------------- -[Alice]: Hello, who's there? --------------- - -Enter a blank line to receive updates, or some text to be written. - -``` - -For anyone else to write to the same register they need to know its xor address, so to communicate -with her friend Bob, Alice needs to find a way to send it to Bob. In her terminal, this is the -value starting "50f4..." in the output above. This value will be different each time you run the -example to create a register. - -Having received the xor address, in another terminal Bob can access the same register to see the -message Alice has written, and he can write back by running this command with the address received -from Alice. (Note that the command should all be on one line): - -``` -cargo run --example registers --features=local -- --user bob --reg-address 50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d -``` - -After retrieving the register and displaying the message from Alice, Bob can reply and at any time, -Alice or Bob can send another message and see any new messages which have been written, or enter a -blank line to poll for updates. - -Here's Bob writing from his terminal: - -``` -Latest value (more than one if concurrent writes were made): --------------- -[Alice]: Hello, who's there? --------------- - -Enter a blank line to receive updates, or some text to be written. -hi Alice, this is Bob! +cargo run --bin ant -- --local file download ``` -Alice will see Bob's message when she either enters a blank line or writes another message herself. - -### Inspect a Register - -A second example, `register_inspect` allows you to view its structure and content. To use this with -the above example you again provide the address of the register. For example: - -``` -cargo run --example register_inspect --features=local -- --reg-address 50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d -``` - -After printing a summary of the register, this example will display -the structure of the register each time you press Enter, including the following: - -``` -Enter a blank line to print the latest register structure (or 'Q' to quit) - -Syncing with SAFE... -synced! -====================== -Root (Latest) Node(s): -[ 0] Node("4eadd9"..) Entry("[alice]: this is alice 3") -[ 3] Node("f05112"..) Entry("[bob]: this is bob 3") -====================== -Register Structure: -(In general, earlier nodes are more indented) -[ 0] Node("4eadd9"..) Entry("[alice]: this is alice 3") - [ 1] Node("f5afb2"..) Entry("[alice]: this is alice 2") - [ 2] Node("7693eb"..) Entry("[alice]: hello this is alice") -[ 3] Node("f05112"..) Entry("[bob]: this is bob 3") - [ 4] Node("8c3cce"..) Entry("[bob]: this is bob 2") - [ 5] Node("c7f9fc"..) Entry("[bob]: this is bob 1") - [ 1] Node("f5afb2"..) Entry("[alice]: this is alice 2") - [ 2] Node("7693eb"..) Entry("[alice]: hello this is alice") -====================== -``` - -Each increase in indentation shows the children of the node above. -The numbers in square brackets are just to make it easier to see -where a node occurs more than once. - ### RPC The node manager launches each node process with a remote procedure call (RPC) service. The diff --git a/ant-bootstrap/Cargo.toml b/ant-bootstrap/Cargo.toml index 856a44fdc0..e07357939b 100644 --- a/ant-bootstrap/Cargo.toml +++ b/ant-bootstrap/Cargo.toml @@ -7,20 +7,20 @@ license = "GPL-3.0" name = "ant-bootstrap" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.4" +version = "0.1.5" [features] local = [] [dependencies] -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } atomic-write-file = "0.2.2" chrono = { version = "0.4", features = ["serde"] } clap = { version = "4.2.1", features = ["derive", "env"] } dirs-next = "~2.0.0" futures = "0.3.30" -libp2p = { version = "0.54.1", features = ["serde"] } +libp2p = { version = "0.55.0", features = ["serde"] } reqwest = { version = "0.12.2", default-features = false, features = [ "rustls-tls-manual-roots", ] } @@ -36,6 +36,3 @@ wiremock = "0.5" tokio = { version = "1.0", features = ["full", "test-util"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } tempfile = "3.8.1" - -[target.'cfg(target_arch = "wasm32")'.dependencies] -wasmtimer = "0.2.0" diff --git a/ant-bootstrap/src/cache_store.rs b/ant-bootstrap/src/cache_store.rs index cb3732148c..760c243b1a 100644 --- a/ant-bootstrap/src/cache_store.rs +++ b/ant-bootstrap/src/cache_store.rs @@ -148,7 +148,7 @@ impl BootstrapCacheStore { &self.config } - /// Create a empty CacheStore with the given configuration + /// Create an empty CacheStore with the given configuration pub fn new(config: BootstrapCacheConfig) -> Result { info!("Creating new CacheStore with config: {:?}", config); let cache_path = config.cache_file_path.clone(); @@ -172,7 +172,7 @@ impl BootstrapCacheStore { Ok(store) } - /// Create a empty CacheStore from the given peers argument. + /// Create an empty CacheStore from the given peers argument. /// This also modifies the cfg if provided based on the PeersArgs. /// And also performs some actions based on the PeersArgs. /// @@ -184,13 +184,13 @@ impl BootstrapCacheStore { let mut config = if let Some(cfg) = config { cfg } else { - BootstrapCacheConfig::default_config()? + BootstrapCacheConfig::default_config(peers_arg.local)? }; if let Some(bootstrap_cache_path) = peers_arg.get_bootstrap_cache_path()? { config.cache_file_path = bootstrap_cache_path; } - let mut store = Self::new(config)?; + let store = Self::new(config)?; // If it is the first node, clear the cache. if peers_arg.first { @@ -198,12 +198,6 @@ impl BootstrapCacheStore { store.write()?; } - // If local mode is enabled, return empty store (will use mDNS) - if peers_arg.local || cfg!(feature = "local") { - info!("Setting config to not write to cache, as 'local' mode is enabled"); - store.config.disable_cache_writing = true; - } - Ok(store) } diff --git a/ant-bootstrap/src/config.rs b/ant-bootstrap/src/config.rs index b81c6377d8..6e5d1d3782 100644 --- a/ant-bootstrap/src/config.rs +++ b/ant-bootstrap/src/config.rs @@ -50,16 +50,18 @@ pub struct BootstrapCacheConfig { impl BootstrapCacheConfig { /// Creates a new BootstrapConfig with default settings - pub fn default_config() -> Result { + /// + /// When `local` is set to true, a different cache file name is used. + /// I.e. the file name will include `_local_` in the name. + pub fn default_config(local: bool) -> Result { + let cache_file_path = if local { + default_cache_path_local()? + } else { + default_cache_path()? + }; Ok(Self { - addr_expiry_duration: ADDR_EXPIRY_DURATION, - max_peers: MAX_PEERS, - max_addrs_per_peer: MAX_ADDRS_PER_PEER, - cache_file_path: default_cache_path()?, - disable_cache_writing: false, - min_cache_save_duration: MIN_BOOTSTRAP_CACHE_SAVE_INTERVAL, - max_cache_save_duration: MAX_BOOTSTRAP_CACHE_SAVE_INTERVAL, - cache_save_scaling_factor: 2, + cache_file_path, + ..Self::empty() }) } @@ -110,6 +112,16 @@ impl BootstrapCacheConfig { /// Returns the default path for the bootstrap cache file fn default_cache_path() -> Result { + Ok(default_cache_dir()?.join(cache_file_name())) +} +/// Returns the default path for the bootstrap cache file that is used for +/// local networks +fn default_cache_path_local() -> Result { + Ok(default_cache_dir()?.join(cache_file_name_local())) +} + +/// Returns the default dir that should contain the bootstrap cache file +fn default_cache_dir() -> Result { let dir = dirs_next::data_dir() .ok_or_else(|| Error::CouldNotObtainDataDir)? .join("autonomi") @@ -117,12 +129,18 @@ fn default_cache_path() -> Result { std::fs::create_dir_all(&dir)?; - let path = dir.join(cache_file_name()); - - Ok(path) + Ok(dir) } /// Returns the name of the cache file pub fn cache_file_name() -> String { format!("bootstrap_cache_{}.json", crate::get_network_version()) } + +/// Returns the name of the cache file for local networks +pub fn cache_file_name_local() -> String { + format!( + "bootstrap_cache_local_{}.json", + crate::get_network_version() + ) +} diff --git a/ant-bootstrap/src/contacts.rs b/ant-bootstrap/src/contacts.rs index d4172fc38d..74ba67b2aa 100644 --- a/ant-bootstrap/src/contacts.rs +++ b/ant-bootstrap/src/contacts.rs @@ -23,11 +23,10 @@ const MAINNET_CONTACTS: &[&str] = &[ ]; /// The client fetch timeout -#[cfg(not(target_arch = "wasm32"))] const FETCH_TIMEOUT_SECS: u64 = 30; /// Maximum number of endpoints to fetch at a time const MAX_CONCURRENT_FETCHES: usize = 3; -/// The max number of retries for a endpoint on failure. +/// The max number of retries for an endpoint on failure. const MAX_RETRIES_ON_FETCH_FAILURE: usize = 3; /// Discovers initial peers from a list of endpoints @@ -50,13 +49,9 @@ impl ContactsFetcher { /// Create a new struct with the provided endpoints pub fn with_endpoints(endpoints: Vec) -> Result { - #[cfg(not(target_arch = "wasm32"))] let request_client = Client::builder() .timeout(Duration::from_secs(FETCH_TIMEOUT_SECS)) .build()?; - // Wasm does not have the timeout method yet. - #[cfg(target_arch = "wasm32")] - let request_client = Client::builder().build()?; Ok(Self { max_addrs: usize::MAX, @@ -218,16 +213,13 @@ impl ContactsFetcher { "Failed to get bootstrap addrs from URL, retrying {retries}/{MAX_RETRIES_ON_FETCH_FAILURE}" ); - #[cfg(not(target_arch = "wasm32"))] tokio::time::sleep(Duration::from_secs(1)).await; - #[cfg(target_arch = "wasm32")] - wasmtimer::tokio::sleep(Duration::from_secs(1)).await; }; Ok(bootstrap_addresses) } - /// Try to parse a response from a endpoint + /// Try to parse a response from an endpoint fn try_parse_response(response: &str, ignore_peer_id: bool) -> Result> { match serde_json::from_str::(response) { Ok(json_endpoints) => { diff --git a/ant-bootstrap/src/initial_peers.rs b/ant-bootstrap/src/initial_peers.rs index 27e59d899c..f323af2796 100644 --- a/ant-bootstrap/src/initial_peers.rs +++ b/ant-bootstrap/src/initial_peers.rs @@ -53,9 +53,7 @@ pub struct PeersArgs { /// a bootstrap cache JSON file. #[clap(long, conflicts_with = "first", value_delimiter = ',')] pub network_contacts_url: Vec, - /// Set to indicate this is a local network. You could also set the `local` feature flag to set this to true. - /// - /// This would use mDNS for peer discovery. + /// Set to indicate this is a local network. #[clap(long, conflicts_with = "network_contacts_url", default_value = "false")] pub local: bool, /// Set to indicate this is a testnet. @@ -116,12 +114,6 @@ impl PeersArgs { return Ok(bootstrap_addresses); } - // If local mode is enabled, return empty store (will use mDNS) - if self.local || cfg!(feature = "local") { - info!("Local mode enabled, using only local discovery."); - return Ok(vec![]); - } - // Add addrs from arguments if present for addr in &self.addrs { if let Some(addr) = craft_valid_multiaddr(addr, false) { @@ -146,7 +138,7 @@ impl PeersArgs { let cfg = if let Some(config) = config { Some(config) } else { - BootstrapCacheConfig::default_config().ok() + BootstrapCacheConfig::default_config(self.local).ok() }; if let Some(mut cfg) = cfg { if let Some(file_path) = self.get_bootstrap_cache_path()? { @@ -177,7 +169,7 @@ impl PeersArgs { } // If we have a network contacts URL, fetch addrs from there. - if !self.network_contacts_url.is_empty() { + if !self.local && !self.network_contacts_url.is_empty() { info!( "Fetching bootstrap address from network contacts URLs: {:?}", self.network_contacts_url @@ -204,7 +196,7 @@ impl PeersArgs { } } - if !self.disable_mainnet_contacts { + if !self.local && !self.disable_mainnet_contacts { let mut contacts_fetcher = ContactsFetcher::with_mainnet_endpoints()?; if let Some(count) = count { contacts_fetcher.set_max_addrs(count); diff --git a/ant-bootstrap/tests/cli_integration_tests.rs b/ant-bootstrap/tests/cli_integration_tests.rs index 98341ae452..ef36457915 100644 --- a/ant-bootstrap/tests/cli_integration_tests.rs +++ b/ant-bootstrap/tests/cli_integration_tests.rs @@ -109,40 +109,6 @@ async fn test_network_contacts_fallback() -> Result<(), Box Result<(), Box> { - let _guard = LogBuilder::init_single_threaded_tokio_test("cli_integration_tests", false); - - let temp_dir = TempDir::new()?; - let cache_path = temp_dir.path().join("cache.json"); - - // Create a config with some peers in the cache - let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - - // Create args with local mode enabled - let args = PeersArgs { - first: false, - addrs: vec![], - network_contacts_url: vec![], - local: true, - disable_mainnet_contacts: false, - ignore_cache: false, - bootstrap_cache_dir: None, - }; - - let addrs = args.get_addrs(Some(config), None).await?; - - assert!(addrs.is_empty(), "Local mode should have no peers"); - - // Verify cache was not touched - assert!( - !cache_path.exists(), - "Cache file should not exist in local mode" - ); - - Ok(()) -} - #[tokio::test] async fn test_test_network_peers() -> Result<(), Box> { let _guard = LogBuilder::init_single_threaded_tokio_test("cli_integration_tests", false); diff --git a/ant-build-info/Cargo.toml b/ant-build-info/Cargo.toml index f664ce5419..81b07d37f0 100644 --- a/ant-build-info/Cargo.toml +++ b/ant-build-info/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-build-info" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.23" +version = "0.1.24" build = "build.rs" include = ["Cargo.toml", "src/**/*", "build.rs"] diff --git a/ant-build-info/src/release_info.rs b/ant-build-info/src/release_info.rs index a05a19a3f1..7595767a34 100644 --- a/ant-build-info/src/release_info.rs +++ b/ant-build-info/src/release_info.rs @@ -1,4 +1,4 @@ -pub const RELEASE_YEAR: &str = "2024"; -pub const RELEASE_MONTH: &str = "12"; -pub const RELEASE_CYCLE: &str = "1"; -pub const RELEASE_CYCLE_COUNTER: &str = "11"; +pub const RELEASE_YEAR: &str = "2025"; +pub const RELEASE_MONTH: &str = "1"; +pub const RELEASE_CYCLE: &str = "2"; +pub const RELEASE_CYCLE_COUNTER: &str = "3"; diff --git a/ant-cli/Cargo.toml b/ant-cli/Cargo.toml index ce02128536..7b676d561e 100644 --- a/ant-cli/Cargo.toml +++ b/ant-cli/Cargo.toml @@ -3,7 +3,7 @@ authors = ["MaidSafe Developers "] name = "ant-cli" description = "CLI client for the Autonomi network" license = "GPL-3.0" -version = "0.3.6" +version = "0.3.7" edition = "2021" homepage = "https://maidsafe.net" readme = "README.md" @@ -15,7 +15,6 @@ path = "src/main.rs" [features] default = ["metrics"] -local = ["ant-bootstrap/local", "autonomi/local", "ant-logging/process-metrics"] metrics = ["ant-logging/process-metrics"] nightly = [] @@ -24,18 +23,13 @@ name = "files" harness = false [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } -autonomi = { path = "../autonomi", version = "0.3.5", features = [ - "fs", - "vault", - "registers", - "loud", -]} +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } +autonomi = { path = "../autonomi", version = "0.3.6", features = [ "loud" ] } clap = { version = "4.2.1", features = ["derive"] } -color-eyre = "~0.6" +color-eyre = "0.6.3" const-hex = "1.13.1" dirs-next = "~2.0.0" hex = "0.4.3" @@ -55,12 +49,12 @@ tokio = { version = "1.32.0", features = [ "sync", "time", "fs", -]} +] } tracing = { version = "~0.1.26" } walkdir = "2.5.0" [dev-dependencies] -autonomi = { path = "../autonomi", version = "0.3.5", features = ["fs"]} +autonomi = { path = "../autonomi", version = "0.3.6" } criterion = "0.5.1" eyre = "0.6.8" rand = { version = "~0.8.5", features = ["small_rng"] } diff --git a/ant-cli/README.md b/ant-cli/README.md index c8c57392ad..babb6736f2 100644 --- a/ant-cli/README.md +++ b/ant-cli/README.md @@ -1,87 +1,258 @@ -# A CLI for the Autonomi Network +# CLI for the Autonomi Network +## Usage ``` -Usage: ant [OPTIONS] +ant [OPTIONS] +``` +### Options +- `--log-output-dest `: Specify the logging output destination. [default: data-dir] +- `--log-format `: Specify the logging format. +- `--peer `: Peer(s) to use for bootstrap, in a 'multiaddr' format containing the peer ID [env: ANT_PEERS=] +- `--timeout `: The maximum duration to wait for a connection to the network before timing out +- `-x, --no-verify`: Prevent verification of data storage on the network +- `-h, --help`: Print help (see more with '--help') +- `-V, --version`: Print version -Commands: - file Operations related to file handling - register Operations related to register management - vault Operations related to vault management - wallet Operations related to wallet management - help Print this message or the help of the given subcommand(s) +## Commands -Options: - --log-output-dest - Specify the logging output destination. [default: data-dir] - --log-format - Specify the logging format. - --peer - Peer(s) to use for bootstrap, in a 'multiaddr' format containing the peer ID [env: ANT_PEERS=] - --timeout - The maximum duration to wait for a connection to the network before timing out - -x, --no-verify - Prevent verification of data storage on the network - -h, --help - Print help (see more with '--help') - -V, --version - Print version -``` +### File +- `file cost ` +- `file upload [--public]` +- `file download ` +- `file list` + +[Reference : File](#file-operations) + +### Register +- `register generate-key [--overwrite]` +- `register cost ` +- `register create ` +- `register edit [--name]
` +- `register get [--name]
` +- `register list` + +### Vault +- `vault cost` +- `vault create` +- `vault load` +- `vault sync [--force]` + +[Reference : Vault](#vault-operations) + +### Wallet +- `wallet create [--no-password] [--password ]` +- `wallet import [--no-password] [--password ]` +- `wallet balance` +- `wallet export` + +[Reference : Wallet](#wallet-operations) + +### Help +- `help` +- `help ` + + +## Installation +You can install the Autonomi CLI in two ways: by directly downloading the binary from GitHub or by building it from source using a terminal. -## Wallet +### Option 1: Downloading the Binary from GitHub -### Create a new wallet +1. Go to the [Releases](https://github.com/maidsafe/autonomi/releases) page on GitHub. +2. Download the latest release for your operating system. +3. Extract the downloaded archive. +4. Move the binary to a directory included in your system's PATH. +### Option 2: Build locally + +1. Ensure you have Rust and Cargo installed on your machine. You can download them from rust-lang.org +2. Clone the repository +``` +git clone https://github.com/maidsafe/autonomi.git +cd autonomi +``` +3. Build the CLI: +``` +cargo build --release --bin=ant +``` +4. Add the CLI to your PATH / Environment Variables +#### Windows (PowerShell) +```powershell +$env:PATH += ";C:\path\to\your\binary" +[System.Environment]::SetEnvironmentVariable("PATH", $env:PATH + ";C:\path\to\your\binary", [System.EnvironmentVariableTarget]::User) +``` + +#### macOS and Linux (Bash) ```bash -wallet create +export PATH=$PATH:/path/to/your/binary +echo 'export PATH=$PATH:/path/to/your/binary' >> ~/.bashrc +source ~/.bashrc ``` -> Add the `--no-password` flag to skip the optional encryption step. +## Reference -> **Wallet Security** -> -> Encrypted wallets provide an additional layer of security, requiring a password to read the private key and perform -> transactions. However, ensure you remember your password; losing it may result in the inability to access your encrypted -> wallet. +### Specify the logging output destination. +``` +--log-output-dest +``` -Example: +Default value: `data-dir`\ +Valid values: [`stdout` , `data-dir` , ] - ```bash - $ wallet create - Enter password (leave empty for none): - Repeat password: - Wallet address: 0xaf676aC7C821977506AC9DcE28bFe83fb06938d8 - Stored wallet in: "/Users/macuser/Library/Application Support/autonomi/client/wallets/0xaf676aC7C821977506AC9DcE28bFe83fb06938d8.encrypted" - ``` +The data directory location is platform specific: +| OS | Path | +| ------------- |:-------------:| +| Linux | $HOME/.local/share/autonomi/client/logs | +| macOS | $HOME/Library/Application Support/autonomi/client/logs | +| Windows | %AppData%\autonomi\client\logs | -### Import a wallet +### Specify the logging format. +``` +--log-format +``` +Valid values [`default` , `json`] -```bash -wallet create --private-key +If the argument is not used, the default format will be applied. + +### Specify the Connection Timeout ``` +--timeout +``` -### Check wallet balance +Default value: `120`\ +Valid values: [`0 - 999`] -```bash -wallet balance +The maximum duration to wait for a connection to the network before timing out.\ +This value is expressed in seconds. + +### Prevent verification of data storage on the network. ``` +-x, --no-verify +``` +This may increase operation speed, but offers no guarantees that operations were successful. -Example: - ```bash - $ wallet balance - Wallet balances: 0x5A631e17FfB0F07b00D88E0e42246495Bf21d698 - +---------------+---+ - | Token Balance | 0 | - +---------------+---+ - | Gas Balance | 0 | - +---------------+---+ - ``` +### File Operations -## License +#### Get a cost estimate for storing a file +``` +file cost +``` + +Gets a cost estimate for uploading a file to the network. +This returns both the storage costs and gas fees for the file. + +Expected value: +- ``: File path (accessible by current user) + + +#### Upload a file +``` +file upload [--public] +``` +Uploads a file to the network. + +Expected value: +- ``: File path (accessible by current user) + +The following flag can be added: +`--public` (Optional) Specifying this will make this file publicly available to anyone on the network + +#### Download a file +``` +file download +``` +Download a file from network address to output path + +Expected values: +- ``: The network address of a file +- ``: The output path to download the file to -This Safe Network repository is licensed under the General Public License (GPL), version -3 ([LICENSE](LICENSE) http://www.gnu.org/licenses/gpl-3.0.en.html). ---- +#### List the files in a vault +``` +file list +``` +Lists all files (both public and private) in a vault. + + +### Vault Operations + +#### Get a cost estimate for storing a vault on the network +``` +vault cost +``` +Gets a cost estimate for uploading a vault to the network. +This returns both the storage costs and gas fees for the vault. + +#### Create a new vault and upload to the network +``` +vault create +``` +Creates a new vault and uploads it to the network. +This will initialise a new vault in the local storage and then upload it to the network. + +#### Load vault from the network +``` +vault load +``` +Retrieves data from the network and writes it to local storage. +This will download the vault data from the network and synchronise it with the local storage. + +#### Sync local data with the network +``` +vault sync [--force] +``` +Sync the users local data with the network vault data. + +The following flag can be applied: +`--force` (Optional) Add this flag to overwrite data in the vault with local user data + +### Wallet Operations +#### Create a new wallet +``` +wallet create [--no-password] +``` + +You will be prompted for an optional password, ignoring this will not encrypt the wallet. +This will output the private key for the wallet, the public key for the wallet, and the stored location on device. + +The following flags can be used to explictly include or exclude encryption of the created wallet + +`--no-password` (Optional) Add this flag to skip the password prompt and encryption step. \ +`--password ` (Optional) Add this flag to encrypt the create wallet + +Note on wallet security +Encrypted wallets provide an additional layer of security, requiring a password to read the private key and perform transactions. However, ensure you remember your password; losing it may result in the inability to access your encrypted wallet. + +#### Imports an existing wallet from a private key +``` +wallet import +``` + +The following flags can be used to explictly include or exclude encryption of the imported wallet + +`--no-password` (Optional) Add this flag to skip the password prompt and encryption step. \ +`--password ` (Optional) Add this flag to encrypt the create wallet + + +#### Displays the wallet balance +``` +wallet balance +``` +This will display both the token and gas balances. + +#### Display the wallet details +``` +wallet export +``` +This will display both the address and private key of the wallet. + + +## Error Handling +If you encounter any errors while using the CLI, you can use the `--log-output-dest` and `--log-format` options to specify logging details. This can help with debugging and understanding the behavior of the CLI. + +## License +This Safe Network repository is licensed under the General Public License (GPL), version 3 (LICENSE http://www.gnu.org/licenses/gpl-3.0.en.html). -Feel free to modify or expand upon this README as needed. Would you like to add or change anything else? +## Contributing +Contributions are welcome! Please read the [CONTRIBUTING.md](https://github.com/maidsafe/autonomi/blob/main/CONTRIBUTING.md) file for guidelines on how to contribute to this project. diff --git a/ant-cli/src/access/keys.rs b/ant-cli/src/access/keys.rs index cfaa5284b7..b1bf834ed9 100644 --- a/ant-cli/src/access/keys.rs +++ b/ant-cli/src/access/keys.rs @@ -7,9 +7,9 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::wallet::load_wallet_private_key; -use autonomi::client::registers::RegisterSecretKey; +use autonomi::client::register::SecretKey as RegisterSecretKey; use autonomi::client::vault::VaultSecretKey; -use autonomi::{get_evm_network_from_env, Wallet}; +use autonomi::{Network, Wallet}; use color_eyre::eyre::{eyre, Context, Result}; use color_eyre::Section; use std::env; @@ -22,11 +22,10 @@ const REGISTER_SIGNING_KEY_ENV: &str = "REGISTER_SIGNING_KEY"; const REGISTER_SIGNING_KEY_FILE: &str = "register_signing_key"; /// EVM wallet -pub fn load_evm_wallet_from_env() -> Result { +pub fn load_evm_wallet_from_env(evm_network: &Network) -> Result { let secret_key = get_secret_key_from_env().wrap_err("The secret key is required to perform this action")?; - let network = get_evm_network_from_env()?; - let wallet = Wallet::new_from_private_key(network, &secret_key) + let wallet = Wallet::new_from_private_key(evm_network.clone(), &secret_key) .wrap_err("Failed to load EVM wallet from key")?; Ok(wallet) } diff --git a/ant-cli/src/access/network.rs b/ant-cli/src/access/network.rs index 8c428e06d3..f69c7a3351 100644 --- a/ant-cli/src/access/network.rs +++ b/ant-cli/src/access/network.rs @@ -12,9 +12,35 @@ use color_eyre::eyre::Context; use color_eyre::Result; use color_eyre::Section; -pub async fn get_peers(peers: PeersArgs) -> Result> { - peers.get_addrs(None, Some(100)).await +pub enum NetworkPeers { + Local(Vec), + Public(Vec), +} + +impl NetworkPeers { + pub fn peers(&self) -> &Vec { + match self { + NetworkPeers::Local(addrs) => addrs, + NetworkPeers::Public(addrs) => addrs, + } + } + + pub fn is_local(&self) -> bool { + matches!(self, NetworkPeers::Local(_)) + } +} + +pub async fn get_peers(peers: PeersArgs) -> Result { + let addrs = peers.get_addrs(None, Some(100)).await .wrap_err("Please provide valid Network peers to connect to") .with_suggestion(|| format!("make sure you've provided network peers using the --peers option or the {ANT_PEERS_ENV} env var")) - .with_suggestion(|| "a peer address looks like this: /ip4/42.42.42.42/udp/4242/quic-v1/p2p/B64nodePeerIDvdjb3FAJF4ks3moreBase64CharsHere") + .with_suggestion(|| "a peer address looks like this: /ip4/42.42.42.42/udp/4242/quic-v1/p2p/B64nodePeerIDvdjb3FAJF4ks3moreBase64CharsHere")?; + + let net = if peers.local { + NetworkPeers::Local(addrs) + } else { + NetworkPeers::Public(addrs) + }; + + Ok(net) } diff --git a/ant-cli/src/access/user_data.rs b/ant-cli/src/access/user_data.rs index 3fc20785cd..2fa3822066 100644 --- a/ant-cli/src/access/user_data.rs +++ b/ant-cli/src/access/user_data.rs @@ -10,16 +10,14 @@ use std::collections::HashMap; use autonomi::client::{ address::{addr_to_str, str_to_addr}, - files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, - registers::{RegisterAddress, RegisterSecretKey}, + files::archive_private::PrivateArchiveAccess, + files::archive_public::ArchiveAddr, + register::RegisterAddress, vault::UserData, }; use color_eyre::eyre::Result; -use super::{ - data_dir::get_client_data_dir_path, - keys::{create_register_signing_key_file, get_register_signing_key}, -}; +use super::data_dir::get_client_data_dir_path; use serde::{Deserialize, Serialize}; @@ -30,16 +28,14 @@ struct PrivateFileArchive { } pub fn get_local_user_data() -> Result { - let register_sk = get_register_signing_key().map(|k| k.to_hex()).ok(); - let registers = get_local_registers()?; let file_archives = get_local_public_file_archives()?; let private_file_archives = get_local_private_file_archives()?; + let registers = get_local_registers()?; let user_data = UserData { - register_sk, - registers, file_archives, private_file_archives, + register_addresses: registers, }; Ok(user_data) } @@ -98,6 +94,15 @@ pub fn get_local_registers() -> Result> { Ok(registers) } +pub fn get_name_of_local_register_with_address(address: &RegisterAddress) -> Result { + let data_dir = get_client_data_dir_path()?; + let user_data_path = data_dir.join("user_data"); + let registers_path = user_data_path.join("registers"); + let file_path = registers_path.join(address.to_hex()); + let file_content = std::fs::read_to_string(file_path)?; + Ok(file_content) +} + pub fn get_local_public_file_archives() -> Result> { let data_dir = get_client_data_dir_path()?; let user_data_path = data_dir.join("user_data"); @@ -119,15 +124,6 @@ pub fn get_local_public_file_archives() -> Result> } pub fn write_local_user_data(user_data: &UserData) -> Result<()> { - if let Some(register_key) = &user_data.register_sk { - let sk = RegisterSecretKey::from_hex(register_key)?; - create_register_signing_key_file(sk)?; - } - - for (register, name) in user_data.registers.iter() { - write_local_register(register, name)?; - } - for (archive, name) in user_data.file_archives.iter() { write_local_public_file_archive(addr_to_str(*archive), name)?; } @@ -136,6 +132,10 @@ pub fn write_local_user_data(user_data: &UserData) -> Result<()> { write_local_private_file_archive(archive.to_hex(), archive.address(), name)?; } + for (register, name) in user_data.register_addresses.iter() { + write_local_register(register, name)?; + } + Ok(()) } diff --git a/ant-cli/src/actions/connect.rs b/ant-cli/src/actions/connect.rs index e7d03fd691..0bdf60040f 100644 --- a/ant-cli/src/actions/connect.rs +++ b/ant-cli/src/actions/connect.rs @@ -6,26 +6,51 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use autonomi::Multiaddr; -use autonomi::{get_evm_network_from_env, Client}; +use crate::network::NetworkPeers; +use autonomi::client::config::ClientOperatingStrategy; +use autonomi::{get_evm_network, Client, ClientConfig}; use color_eyre::eyre::bail; use color_eyre::eyre::Result; use indicatif::ProgressBar; use std::time::Duration; -pub async fn connect_to_network(peers: Vec) -> Result { +pub async fn connect_to_network(peers: NetworkPeers) -> Result { + connect_to_network_with_config(peers, Default::default()).await +} + +pub async fn connect_to_network_with_config( + peers: NetworkPeers, + operation_config: ClientOperatingStrategy, +) -> Result { let progress_bar = ProgressBar::new_spinner(); progress_bar.enable_steady_tick(Duration::from_millis(120)); progress_bar.set_message("Connecting to The Autonomi Network..."); let new_style = progress_bar.style().tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈🔗"); progress_bar.set_style(new_style); - progress_bar.set_message("Connecting to The Autonomi Network..."); + let local = peers.is_local(); + + let peers_opt = if local { + progress_bar.set_message("Connecting to a local Autonomi Network..."); + None + } else { + progress_bar.set_message("Connecting to The Autonomi Network..."); + Some(peers.peers().to_vec()) + }; + + let evm_network = get_evm_network(local)?; + + let config = ClientConfig { + local, + peers: peers_opt, + evm_network, + strategy: operation_config, + }; + + let res = Client::init_with_config(config).await; - match Client::init_with_peers(peers).await { - Ok(mut client) => { - let evm_network = get_evm_network_from_env()?; - client.set_evm_network(evm_network); + match res { + Ok(client) => { info!("Connected to the Network"); progress_bar.finish_with_message("Connected to the Network"); Ok(client) diff --git a/ant-cli/src/actions/download.rs b/ant-cli/src/actions/download.rs index 6b3bbd380c..758c2ba7a4 100644 --- a/ant-cli/src/actions/download.rs +++ b/ant-cli/src/actions/download.rs @@ -9,8 +9,8 @@ use super::get_progress_bar; use autonomi::{ client::{ - address::str_to_addr, - files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, + address::str_to_addr, files::archive_private::PrivateArchiveAccess, + files::archive_public::ArchiveAddr, }, Client, }; @@ -20,7 +20,7 @@ use color_eyre::{ }; use std::path::PathBuf; -pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Result<()> { +pub async fn download(addr: &str, dest_path: &str, client: &Client) -> Result<()> { let public_address = str_to_addr(addr).ok(); let private_address = crate::user_data::get_local_private_archive_access(addr) .inspect_err(|e| error!("Failed to get private archive access: {e}")) @@ -40,10 +40,10 @@ async fn download_private( addr: &str, private_address: PrivateArchiveAccess, dest_path: &str, - client: &mut Client, + client: &Client, ) -> Result<()> { let archive = client - .archive_get(private_address) + .archive_get(&private_address) .await .wrap_err("Failed to fetch data from address")?; @@ -51,7 +51,7 @@ async fn download_private( let mut all_errs = vec![]; for (path, access, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.data_get(access.clone()).await { + let bytes = match client.data_get(access).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); @@ -86,10 +86,10 @@ async fn download_public( addr: &str, address: ArchiveAddr, dest_path: &str, - client: &mut Client, + client: &Client, ) -> Result<()> { let archive = client - .archive_get_public(address) + .archive_get_public(&address) .await .wrap_err("Failed to fetch data from address")?; @@ -97,7 +97,7 @@ async fn download_public( let mut all_errs = vec![]; for (path, addr, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.data_get_public(*addr).await { + let bytes = match client.data_get_public(addr).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); diff --git a/ant-cli/src/actions/mod.rs b/ant-cli/src/actions/mod.rs index 8b4662c3d9..31720cd2b8 100644 --- a/ant-cli/src/actions/mod.rs +++ b/ant-cli/src/actions/mod.rs @@ -10,7 +10,6 @@ mod connect; mod download; mod progress_bar; -pub use connect::connect_to_network; +pub use connect::{connect_to_network, connect_to_network_with_config}; pub use download::download; - pub use progress_bar::get_progress_bar; diff --git a/ant-cli/src/commands.rs b/ant-cli/src/commands.rs index ff065a06c0..c98df9e8de 100644 --- a/ant-cli/src/commands.rs +++ b/ant-cli/src/commands.rs @@ -12,7 +12,8 @@ mod vault; mod wallet; use crate::opt::Opt; -use clap::Subcommand; +use autonomi::ResponseQuorum; +use clap::{error::ErrorKind, CommandFactory as _, Subcommand}; use color_eyre::Result; #[derive(Subcommand, Debug)] @@ -57,6 +58,11 @@ pub enum FileCmd { /// Upload the file as public. Everyone can see public data on the Network. #[arg(short, long)] public: bool, + /// Experimental: Optionally specify the quorum for the verification of the upload. + /// + /// Possible values are: "one", "majority", "all", n (where n is a number greater than 0) + #[arg(short, long)] + quorum: Option, }, /// Download a file from the given address. @@ -65,6 +71,11 @@ pub enum FileCmd { addr: String, /// The destination file path. dest_file: String, + /// Experimental: Optionally specify the quorum for the download (makes sure that we have n copies for each chunks). + /// + /// Possible values are: "one", "majority", "all", n (where n is a number greater than 0) + #[arg(short, long)] + quorum: Option, }, /// List previous uploads @@ -88,17 +99,16 @@ pub enum RegisterCmd { }, /// Create a new register with the given name and value. + /// Note that anyone with the register address can read its value. Create { /// The name of the register. name: String, /// The value to store in the register. value: String, - /// Create the register with public write access. - #[arg(long, default_value = "false")] - public: bool, }, /// Edit an existing register. + /// Note that anyone with the register address can read its value. Edit { /// Use the name of the register instead of the address /// Note that only the owner of the register can use this shorthand as the address can be generated from the name and register key. @@ -129,7 +139,11 @@ pub enum RegisterCmd { #[derive(Subcommand, Debug)] pub enum VaultCmd { /// Estimate cost to create a vault. - Cost, + Cost { + /// Expected max_size of a vault, only for cost estimation. + #[clap(default_value = "3145728")] + expected_max_size: u64, + }, /// Create a vault at a deterministic address based on your `SECRET_KEY`. /// Pushing an encrypted backup of your local user data to the network @@ -140,7 +154,7 @@ pub enum VaultCmd { /// You need to have your original `SECRET_KEY` to load the vault. Load, - /// Sync vault with the network, including registers and files. + /// Sync vault with the network, safeguarding local user data. /// Loads existing user data from the network and merges it with your local user data. /// Pushes your local user data to the network. Sync { @@ -189,20 +203,24 @@ pub async fn handle_subcommand(opt: Opt) -> Result<()> { match cmd { Some(SubCmd::File { command }) => match command { FileCmd::Cost { file } => file::cost(&file, peers.await?).await, - FileCmd::Upload { file, public } => file::upload(&file, public, peers.await?).await, - FileCmd::Download { addr, dest_file } => { - file::download(&addr, &dest_file, peers.await?).await - } + FileCmd::Upload { + file, + public, + quorum, + } => file::upload(&file, public, peers.await?, quorum).await, + FileCmd::Download { + addr, + dest_file, + quorum, + } => file::download(&addr, &dest_file, peers.await?, quorum).await, FileCmd::List => file::list(), }, Some(SubCmd::Register { command }) => match command { RegisterCmd::GenerateKey { overwrite } => register::generate_key(overwrite), RegisterCmd::Cost { name } => register::cost(&name, peers.await?).await, - RegisterCmd::Create { - name, - value, - public, - } => register::create(&name, &value, public, peers.await?).await, + RegisterCmd::Create { name, value } => { + register::create(&name, &value, peers.await?).await + } RegisterCmd::Edit { address, name, @@ -212,10 +230,12 @@ pub async fn handle_subcommand(opt: Opt) -> Result<()> { RegisterCmd::List => register::list(), }, Some(SubCmd::Vault { command }) => match command { - VaultCmd::Cost => vault::cost(peers.await?).await, + VaultCmd::Cost { expected_max_size } => { + vault::cost(peers.await?, expected_max_size).await + } VaultCmd::Create => vault::create(peers.await?).await, VaultCmd::Load => vault::load(peers.await?).await, - VaultCmd::Sync { force } => vault::sync(peers.await?, force).await, + VaultCmd::Sync { force } => vault::sync(force, peers.await?).await, }, Some(SubCmd::Wallet { command }) => match command { WalletCmd::Create { @@ -228,8 +248,13 @@ pub async fn handle_subcommand(opt: Opt) -> Result<()> { password, } => wallet::import(private_key, no_password, password), WalletCmd::Export => wallet::export(), - WalletCmd::Balance => wallet::balance().await, + WalletCmd::Balance => wallet::balance(peers.await?.is_local()).await, }, - None => Ok(()), + None => { + // If no subcommand is given, default to clap's error behaviour. + Opt::command() + .error(ErrorKind::MissingSubcommand, "Please provide a subcommand") + .exit(); + } } } diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index 146133e348..d9f52c6a13 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -6,16 +6,18 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::network::NetworkPeers; use crate::utils::collect_upload_summary; use crate::wallet::load_wallet; use autonomi::client::address::addr_to_str; -use autonomi::Multiaddr; +use autonomi::ClientOperatingStrategy; +use autonomi::ResponseQuorum; use color_eyre::eyre::Context; use color_eyre::eyre::Result; use color_eyre::Section; use std::path::PathBuf; -pub async fn cost(file: &str, peers: Vec) -> Result<()> { +pub async fn cost(file: &str, peers: NetworkPeers) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; println!("Getting upload cost..."); @@ -31,9 +33,19 @@ pub async fn cost(file: &str, peers: Vec) -> Result<()> { Ok(()) } -pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<()> { - let wallet = load_wallet()?; - let mut client = crate::actions::connect_to_network(peers).await?; +pub async fn upload( + file: &str, + public: bool, + peers: NetworkPeers, + optional_verification_quorum: Option, +) -> Result<()> { + let mut config = ClientOperatingStrategy::new(); + if let Some(verification_quorum) = optional_verification_quorum { + config.chunks.verification_quorum = verification_quorum; + } + let mut client = crate::actions::connect_to_network_with_config(peers, config).await?; + + let wallet = load_wallet(client.evm_network())?; let event_receiver = client.enable_client_events(); let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver); @@ -52,14 +64,14 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( // upload dir let local_addr; let archive = if public { - let xor_name = client + let (_cost, xor_name) = client .dir_and_archive_upload_public(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; local_addr = addr_to_str(xor_name); local_addr.clone() } else { - let private_data_access = client + let (_cost, private_data_access) = client .dir_and_archive_upload(dir_path, &wallet) .await .wrap_err("Failed to upload dir and archive")?; @@ -76,13 +88,17 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( // get summary let summary = upload_summary_thread.await?; - if summary.record_count == 0 { + if summary.records_paid == 0 { println!("All chunks already exist on the network."); } else { println!("Successfully uploaded: {file}"); println!("At address: {local_addr}"); info!("Successfully uploaded: {file} at address: {local_addr}"); - println!("Number of chunks uploaded: {}", summary.record_count); + println!("Number of chunks uploaded: {}", summary.records_paid); + println!( + "Number of chunks already paid/uploaded: {}", + summary.records_already_paid + ); println!("Total cost: {} AttoTokens", summary.tokens_spent); } info!("Summary for upload of file {file} at {local_addr:?}: {summary:?}"); @@ -101,9 +117,18 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( Ok(()) } -pub async fn download(addr: &str, dest_path: &str, peers: Vec) -> Result<()> { - let mut client = crate::actions::connect_to_network(peers).await?; - crate::actions::download(addr, dest_path, &mut client).await +pub async fn download( + addr: &str, + dest_path: &str, + peers: NetworkPeers, + quorum: Option, +) -> Result<()> { + let mut config = ClientOperatingStrategy::new(); + if let Some(quorum) = quorum { + config.chunks.get_quorum = quorum; + } + let client = crate::actions::connect_to_network_with_config(peers, config).await?; + crate::actions::download(addr, dest_path, &client).await } pub fn list() -> Result<()> { diff --git a/ant-cli/src/commands/register.rs b/ant-cli/src/commands/register.rs index 5598fc0544..cbd0c0c3ee 100644 --- a/ant-cli/src/commands/register.rs +++ b/ant-cli/src/commands/register.rs @@ -8,13 +8,11 @@ #![allow(deprecated)] -use crate::utils::collect_upload_summary; +use crate::network::NetworkPeers; use crate::wallet::load_wallet; -use autonomi::client::registers::RegisterAddress; -use autonomi::client::registers::RegisterPermissions; -use autonomi::client::registers::RegisterSecretKey; +use autonomi::client::register::RegisterAddress; +use autonomi::client::register::SecretKey as RegisterSecretKey; use autonomi::Client; -use autonomi::Multiaddr; use color_eyre::eyre::eyre; use color_eyre::eyre::Context; use color_eyre::eyre::Result; @@ -39,13 +37,14 @@ pub fn generate_key(overwrite: bool) -> Result<()> { Ok(()) } -pub async fn cost(name: &str, peers: Vec) -> Result<()> { - let register_key = crate::keys::get_register_signing_key() +pub async fn cost(name: &str, peers: NetworkPeers) -> Result<()> { + let main_registers_key = crate::keys::get_register_signing_key() .wrap_err("The register key is required to perform this action")?; let client = crate::actions::connect_to_network(peers).await?; + let key_for_name = Client::register_key_from_name(&main_registers_key, name); let cost = client - .register_cost(name.to_string(), register_key) + .register_cost(&key_for_name.public_key()) .await .wrap_err("Failed to get cost for register")?; info!("Estimated cost to create a register with name {name}: {cost}"); @@ -53,65 +52,28 @@ pub async fn cost(name: &str, peers: Vec) -> Result<()> { Ok(()) } -pub async fn create(name: &str, value: &str, public: bool, peers: Vec) -> Result<()> { - let wallet = load_wallet()?; - let register_key = crate::keys::get_register_signing_key() +pub async fn create(name: &str, value: &str, peers: NetworkPeers) -> Result<()> { + let main_registers_key = crate::keys::get_register_signing_key() .wrap_err("The register key is required to perform this action")?; - let mut client = crate::actions::connect_to_network(peers).await?; - let event_receiver = client.enable_client_events(); - let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver); + let client = crate::actions::connect_to_network(peers).await?; + let wallet = load_wallet(client.evm_network())?; + let register_key = Client::register_key_from_name(&main_registers_key, name); println!("Creating register with name: {name}"); info!("Creating register with name: {name}"); - let register = if public { - println!("With public write access"); - info!("With public write access"); - let permissions = RegisterPermissions::new_anyone_can_write(); - client - .register_create_with_permissions( - Some(value.as_bytes().to_vec().into()), - name, - register_key, - permissions, - &wallet, - ) - .await - .wrap_err("Failed to create register")? - } else { - println!("With private write access"); - info!("With private write access"); - client - .register_create( - Some(value.as_bytes().to_vec().into()), - name, - register_key, - &wallet, - ) - .await - .wrap_err("Failed to create register")? - }; - - let address = register.address(); - - if let Err(e) = upload_completed_tx.send(()) { - error!("Failed to send upload completed event: {e:?}"); - eprintln!("Failed to send upload completed event: {e:?}"); - } + let content = Client::register_value_from_bytes(value.as_bytes())?; + let (cost, address) = client + .register_create(®ister_key, content, wallet.into()) + .await + .wrap_err("Failed to create register")?; - let summary = upload_summary_thread.await?; - if summary.record_count == 0 { - println!("✅ The register already exists on the network at address: {address}."); - println!("No tokens were spent."); - } else { - println!("✅ Register created at address: {address}"); - println!("With name: {name}"); - println!("And initial value: [{value}]"); - info!("Register created at address: {address} with name: {name}"); - println!("Total cost: {} AttoTokens", summary.tokens_spent); - } - info!("Summary of register creation: {summary:?}"); + println!("✅ Register created at address: {address}"); + println!("With name: {name}"); + println!("And initial value: [{value}]"); + info!("Register created at address: {address} with name: {name}"); + println!("Total cost: {cost} AttoTokens"); - crate::user_data::write_local_register(address, name) + crate::user_data::write_local_register(&address, name) .wrap_err("Failed to save register to local user data") .with_suggestion(|| "Local user data saves the register address above to disk, without it you need to keep track of the address yourself")?; info!("Saved register to local user data"); @@ -119,51 +81,54 @@ pub async fn create(name: &str, value: &str, public: bool, peers: Vec Ok(()) } -pub async fn edit(address: String, name: bool, value: &str, peers: Vec) -> Result<()> { - let register_key = crate::keys::get_register_signing_key() +pub async fn edit(address: String, name: bool, value: &str, peers: NetworkPeers) -> Result<()> { + let main_registers_key = crate::keys::get_register_signing_key() .wrap_err("The register key is required to perform this action")?; let client = crate::actions::connect_to_network(peers).await?; + let wallet = load_wallet(client.evm_network())?; + let value_bytes = Client::register_value_from_bytes(value.as_bytes())?; - let address = if name { - Client::register_address(&address, ®ister_key) + let register_key = if name { + let name_str = address.clone(); + Client::register_key_from_name(&main_registers_key, &name_str) } else { - RegisterAddress::from_hex(&address) + let addr = RegisterAddress::from_hex(&address) .wrap_err(format!("Failed to parse register address: {address}")) .with_suggestion(|| { "if you want to use the name as the address, run the command with the --name flag" - })? + })?; + let name_str = crate::user_data::get_name_of_local_register_with_address(&addr) + .wrap_err(format!("Could not find a register with address in local user data: {address}")) + .with_suggestion(|| "This register is not known to this client, try to create it first.") + .with_suggestion(|| "If you indeed have created this register before, retry using its name by using the --name flag")?; + Client::register_key_from_name(&main_registers_key, &name_str) }; - println!("Getting register at address: {address}"); - info!("Getting register at address: {address}"); - let register = client - .register_get(address) - .await - .wrap_err(format!("Failed to get register at address: {address}"))?; - - println!("Found register at address: {address}"); - println!("Updating register with new value: {value}"); - info!("Updating register at address: {address} with new value: {value}"); + println!("Attempting to update register at {address} with new value: {value}"); + info!("Attempting to update register at {address} with new value: {value}"); - client - .register_update(register, value.as_bytes().to_vec().into(), register_key) + let cost = client + .register_update(®ister_key, value_bytes, wallet.into()) .await .wrap_err(format!("Failed to update register at address: {address}"))?; println!("✅ Successfully updated register"); println!("With value: [{value}]"); + println!("Total cost: {cost} AttoTokens"); info!("Successfully updated register at address: {address}"); Ok(()) } -pub async fn get(address: String, name: bool, peers: Vec) -> Result<()> { - let register_key = crate::keys::get_register_signing_key() - .wrap_err("The register key is required to perform this action")?; +pub async fn get(address: String, name: bool, peers: NetworkPeers) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; - let address = if name { - Client::register_address(&address, ®ister_key) + let addr = if name { + let name_str = address.clone(); + let main_registers_key = crate::keys::get_register_signing_key() + .wrap_err("The register key is required to perform this action")?; + let register_key = Client::register_key_from_name(&main_registers_key, &name_str); + RegisterAddress::new(register_key.public_key()) } else { RegisterAddress::from_hex(&address) .wrap_err(format!("Failed to parse register address: {address}")) @@ -172,25 +137,24 @@ pub async fn get(address: String, name: bool, peers: Vec) -> Result<( })? }; - println!("Getting register at address: {address}"); - info!("Getting register at address: {address}"); - let register = client - .register_get(address) - .await - .wrap_err(format!("Failed to get register at address: {address}"))?; - let values = register.values(); - - println!("✅ Register found at address: {address}"); - info!("Register found at address: {address}"); - match values.as_slice() { - [one] => println!("With value: [{:?}]", String::from_utf8_lossy(one)), - _ => { - println!("With multiple concurrent values:"); - for value in values.iter() { - println!("[{:?}]", String::from_utf8_lossy(value)); - } - } + if name { + println!("Getting register with name: {address}"); + info!("Getting register with name: {address}"); + } else { + println!("Getting register at address: {address}"); + info!("Getting register at address: {address}"); } + let value_bytes = client + .register_get(&addr) + .await + .wrap_err(format!("Error getting register at: {address}"))?; + + println!("✅ Register found at: {address}"); + info!("Register found at: {address}"); + let value = String::from_utf8_lossy(&value_bytes); + println!("With value: [{value}]"); + info!("With value: [{value}]"); + Ok(()) } diff --git a/ant-cli/src/commands/vault.rs b/ant-cli/src/commands/vault.rs index b5446f8962..26ab592093 100644 --- a/ant-cli/src/commands/vault.rs +++ b/ant-cli/src/commands/vault.rs @@ -6,38 +6,37 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::network::NetworkPeers; use crate::wallet::load_wallet; -use autonomi::Multiaddr; use color_eyre::eyre::Context; use color_eyre::eyre::Result; use color_eyre::Section; -pub async fn cost(peers: Vec) -> Result<()> { +pub async fn cost(peers: NetworkPeers, expected_max_size: u64) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; let vault_sk = crate::keys::get_vault_secret_key()?; println!("Getting cost to create a new vault..."); - let total_cost = client.vault_cost(&vault_sk).await?; + let total_cost = client.vault_cost(&vault_sk, expected_max_size).await?; if total_cost.is_zero() { - println!("Vault already exists, modifying an existing vault is free"); + println!("Vault already exists, updating an existing vault is free unless the new content exceeds the current vault's paid capacity."); } else { println!("Cost to create a new vault: {total_cost} AttoTokens"); } Ok(()) } -pub async fn create(peers: Vec) -> Result<()> { +pub async fn create(peers: NetworkPeers) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; - let wallet = load_wallet()?; + let wallet = load_wallet(client.evm_network())?; let vault_sk = crate::keys::get_vault_secret_key()?; println!("Retrieving local user data..."); let local_user_data = crate::user_data::get_local_user_data()?; let file_archives_len = local_user_data.file_archives.len(); let private_file_archives_len = local_user_data.private_file_archives.len(); - let registers_len = local_user_data.registers.len(); - + let registers_len = local_user_data.register_addresses.len(); println!("Pushing to network vault..."); let total_cost = client .put_user_data_to_vault(&vault_sk, wallet.into(), local_user_data) @@ -57,21 +56,20 @@ pub async fn create(peers: Vec) -> Result<()> { Ok(()) } -pub async fn sync(peers: Vec, force: bool) -> Result<()> { +pub async fn sync(force: bool, peers: NetworkPeers) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; let vault_sk = crate::keys::get_vault_secret_key()?; - let wallet = load_wallet()?; - - println!("Fetching vault from network..."); - let net_user_data = client - .get_user_data_from_vault(&vault_sk) - .await - .wrap_err("Failed to fetch vault from network") - .with_suggestion(|| "Make sure you have already created a vault on the network")?; + let wallet = load_wallet(client.evm_network())?; if force { println!("The force flag was provided, overwriting user data in the vault with local user data..."); } else { + println!("Fetching vault from network..."); + let net_user_data = client + .get_user_data_from_vault(&vault_sk) + .await + .wrap_err("Failed to fetch vault from network") + .with_suggestion(|| "Make sure you have already created a vault on the network")?; println!("Syncing vault with local user data..."); crate::user_data::write_local_user_data(&net_user_data)?; } @@ -80,10 +78,11 @@ pub async fn sync(peers: Vec, force: bool) -> Result<()> { let local_user_data = crate::user_data::get_local_user_data()?; let file_archives_len = local_user_data.file_archives.len(); let private_file_archives_len = local_user_data.private_file_archives.len(); - let registers_len = local_user_data.registers.len(); + let registers_len = local_user_data.register_addresses.len(); client .put_user_data_to_vault(&vault_sk, wallet.into(), local_user_data) - .await?; + .await + .with_suggestion(|| "Make sure you have already created a vault on the network")?; println!("✅ Successfully synced vault"); println!("Vault contains:"); @@ -93,7 +92,7 @@ pub async fn sync(peers: Vec, force: bool) -> Result<()> { Ok(()) } -pub async fn load(peers: Vec) -> Result<()> { +pub async fn load(peers: NetworkPeers) -> Result<()> { let client = crate::actions::connect_to_network(peers).await?; let vault_sk = crate::keys::get_vault_secret_key()?; @@ -108,6 +107,6 @@ pub async fn load(peers: Vec) -> Result<()> { "{} private file archive(s)", user_data.private_file_archives.len() ); - println!("{} register(s)", user_data.registers.len()); + println!("{} register(s)", user_data.register_addresses.len()); Ok(()) } diff --git a/ant-cli/src/commands/wallet.rs b/ant-cli/src/commands/wallet.rs index b1a2caf70b..9e76128bbe 100644 --- a/ant-cli/src/commands/wallet.rs +++ b/ant-cli/src/commands/wallet.rs @@ -6,9 +6,10 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::wallet::fs::{select_wallet, select_wallet_private_key, store_private_key}; +use crate::wallet::fs::{select_wallet_private_key, store_private_key}; use crate::wallet::input::request_password; use crate::wallet::DUMMY_NETWORK; +use autonomi::get_evm_network; use autonomi::Wallet; use color_eyre::eyre::eyre; use color_eyre::Result; @@ -22,7 +23,7 @@ pub fn create(no_password: bool, password: Option) -> Result<()> { let wallet_private_key = Wallet::random_private_key(); let wallet_address = Wallet::new_from_private_key(DUMMY_NETWORK, &wallet_private_key) - .expect("Infallible") + .map_err(|e| eyre!("Unexpected error: Failed to create wallet from private key: {e}"))? .address() .to_string(); @@ -48,7 +49,7 @@ pub fn import( let maybe_encryption_password = maybe_request_password(no_password, password)?; let wallet_address = Wallet::new_from_private_key(DUMMY_NETWORK, &wallet_private_key) - .expect("Infallible") + .map_err(|e| eyre!("Unexpected error: Failed to create wallet from private key: {e}"))? .address() .to_string(); @@ -70,7 +71,7 @@ pub fn export() -> Result<()> { let wallet_private_key = select_wallet_private_key()?; let wallet_address = Wallet::new_from_private_key(DUMMY_NETWORK, &wallet_private_key) - .expect("Infallible") + .map_err(|e| eyre!("Failed to create wallet from private key loaded from disk: {e}"))? .address() .to_string(); @@ -80,8 +81,9 @@ pub fn export() -> Result<()> { Ok(()) } -pub async fn balance() -> Result<()> { - let wallet = select_wallet()?; +pub async fn balance(local: bool) -> Result<()> { + let network = get_evm_network(local)?; + let wallet = crate::wallet::load_wallet(&network)?; let token_balance = wallet.balance_of_tokens().await?; let gas_balance = wallet.balance_of_gas_tokens().await?; diff --git a/ant-cli/src/main.rs b/ant-cli/src/main.rs index 971c38fd6a..bf821226e3 100644 --- a/ant-cli/src/main.rs +++ b/ant-cli/src/main.rs @@ -24,7 +24,6 @@ pub use access::user_data; use clap::Parser; use color_eyre::Result; -#[cfg(feature = "local")] use ant_logging::metrics::init_metrics; use ant_logging::{LogBuilder, LogFormat, ReloadHandle, WorkerGuard}; use ant_protocol::version; @@ -73,8 +72,9 @@ async fn main() -> Result<()> { } let _log_guards = init_logging_and_metrics(&opt)?; - #[cfg(feature = "local")] - tokio::spawn(init_metrics(std::process::id())); + if opt.peers.local { + tokio::spawn(init_metrics(std::process::id())); + } info!("\"{}\"", std::env::args().collect::>().join(" ")); let version = ant_build_info::git_info(); @@ -91,7 +91,6 @@ fn init_logging_and_metrics(opt: &Opt) -> Result<(ReloadHandle, Option, diff --git a/ant-cli/src/utils.rs b/ant-cli/src/utils.rs index 5f031a3c24..19f02878f9 100644 --- a/ant-cli/src/utils.rs +++ b/ant-cli/src/utils.rs @@ -20,6 +20,7 @@ pub fn collect_upload_summary( let stats_thread = tokio::spawn(async move { let mut tokens_spent: Amount = Amount::from(0); let mut record_count = 0; + let mut records_already_paid = 0; loop { tokio::select! { @@ -27,7 +28,8 @@ pub fn collect_upload_summary( match event { Some(ClientEvent::UploadComplete(upload_summary)) => { tokens_spent += upload_summary.tokens_spent; - record_count += upload_summary.record_count; + record_count += upload_summary.records_paid; + records_already_paid += upload_summary.records_already_paid; } None => break, } @@ -41,14 +43,16 @@ pub fn collect_upload_summary( match event { ClientEvent::UploadComplete(upload_summary) => { tokens_spent += upload_summary.tokens_spent; - record_count += upload_summary.record_count; + record_count += upload_summary.records_paid; + records_already_paid += upload_summary.records_already_paid; } } } UploadSummary { tokens_spent, - record_count, + records_paid: record_count, + records_already_paid, } }); diff --git a/ant-cli/src/wallet/encryption.rs b/ant-cli/src/wallet/encryption.rs index 88f53afa15..bbad0cc2f7 100644 --- a/ant-cli/src/wallet/encryption.rs +++ b/ant-cli/src/wallet/encryption.rs @@ -6,7 +6,8 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::wallet::error::Error; +use color_eyre::eyre::eyre; +use color_eyre::Result; use rand::Rng; use ring::aead::{BoundKey, Nonce, NonceSequence}; use ring::error::Unspecified; @@ -28,7 +29,7 @@ impl NonceSequence for NonceSeq { } } -pub fn encrypt_private_key(private_key: &str, password: &str) -> Result { +pub fn encrypt_private_key(private_key: &str, password: &str) -> Result { // Generate a random salt // Salt is used to ensure unique derived keys even for identical passwords let mut salt = [0u8; SALT_LENGTH]; @@ -55,7 +56,7 @@ pub fn encrypt_private_key(private_key: &str, password: &str) -> Result Result Result Result { +pub fn decrypt_private_key(encrypted_data: &str, password: &str) -> Result { let encrypted_data = hex::decode(encrypted_data) - .map_err(|_| Error::FailedToDecryptKey(String::from("Encrypted data is invalid")))?; + .map_err(|_| eyre!("Failed to decrypt key: Encrypted data is invalid"))?; let salt: [u8; SALT_LENGTH] = encrypted_data[..SALT_LENGTH] .try_into() - .map_err(|_| Error::FailedToDecryptKey(String::from("Could not find salt")))?; + .map_err(|_| eyre!("Failed to decrypt key: Could not find salt"))?; let nonce: [u8; NONCE_LENGTH] = encrypted_data[SALT_LENGTH..SALT_LENGTH + NONCE_LENGTH] .try_into() - .map_err(|_| Error::FailedToDecryptKey(String::from("Could not find nonce")))?; + .map_err(|_| eyre!("Failed to decrypt key: Could not find nonce"))?; let encrypted_private_key = &encrypted_data[SALT_LENGTH + NONCE_LENGTH..]; @@ -106,7 +107,7 @@ pub fn decrypt_private_key(encrypted_data: &str, password: &str) -> Result Result = OnceLock::new(); /// Creates the wallets folder if it is missing and returns the folder path. -pub(crate) fn get_client_wallet_dir_path() -> Result { - let mut home_dirs = dirs_next::data_dir().ok_or(Error::WalletsFolderNotFound)?; - home_dirs.push("autonomi"); - home_dirs.push("client"); +pub(crate) fn get_client_wallet_dir_path() -> Result { + let mut home_dirs = crate::access::data_dir::get_client_data_dir_path() + .wrap_err("Failed to get wallet directory")?; home_dirs.push("wallets"); - std::fs::create_dir_all(home_dirs.as_path()).map_err(|_| Error::FailedToCreateWalletsFolder)?; + std::fs::create_dir_all(home_dirs.as_path()) + .wrap_err("Failed to create wallets folder, make sure you have the correct permissions")?; Ok(home_dirs) } @@ -41,9 +41,9 @@ pub(crate) fn get_client_wallet_dir_path() -> Result { pub(crate) fn store_private_key( private_key: &str, encryption_password: Option, -) -> Result { +) -> Result { let wallet = Wallet::new_from_private_key(DUMMY_NETWORK, private_key) - .map_err(|_| Error::InvalidPrivateKey)?; + .map_err(|_| eyre!("Private key is invalid"))?; // Wallet address let wallet_address = wallet.address().to_string(); @@ -56,15 +56,13 @@ pub(crate) fn store_private_key( let file_name = format!("{wallet_address}{ENCRYPTED_PRIVATE_KEY_EXT}"); let file_path = wallets_folder.join(file_name); - std::fs::write(file_path.clone(), encrypted_key) - .map_err(|err| Error::FailedToStorePrivateKey(err.to_string()))?; + std::fs::write(file_path.clone(), encrypted_key).wrap_err("Failed to store private key")?; Ok(file_path.into_os_string()) } else { let file_path = wallets_folder.join(wallet_address); - std::fs::write(file_path.clone(), private_key) - .map_err(|err| Error::FailedToStorePrivateKey(err.to_string()))?; + std::fs::write(file_path.clone(), private_key).wrap_err("Failed to store private key")?; Ok(file_path.into_os_string()) } @@ -73,7 +71,7 @@ pub(crate) fn store_private_key( /// Loads the private key (hex-encoded) from disk. /// /// If the private key file is encrypted, the function will prompt for the decryption password in the CLI. -pub(crate) fn load_private_key(wallet_address: &str) -> Result { +pub(crate) fn load_private_key(wallet_address: &str) -> Result { let wallets_folder = get_client_wallet_dir_path()?; let mut file_name = wallet_address.to_string(); @@ -93,46 +91,42 @@ pub(crate) fn load_private_key(wallet_address: &str) -> Result { let file_path = wallets_folder.join(file_name); - let mut file = std::fs::File::open(&file_path).map_err(|_| Error::PrivateKeyFileNotFound)?; + let mut file = + std::fs::File::open(&file_path).map_err(|e| eyre!("Private key file not found: {e}"))?; let mut buffer = String::new(); file.read_to_string(&mut buffer) - .map_err(|_| Error::InvalidPrivateKeyFile)?; + .map_err(|_| eyre!("Invalid private key file"))?; // If the file is encrypted, prompt for the password and decrypt the key. if is_encrypted { let password = get_password_input("Enter password to decrypt wallet:"); decrypt_private_key(&buffer, &password) + .map_err(|e| eyre!("Failed to decrypt private key: {e}")) } else { Ok(buffer) } } -pub(crate) fn load_wallet_from_address(wallet_address: &str) -> Result { - let network = get_evm_network_from_env().expect("Could not load EVM network from environment"); +pub(crate) fn load_wallet_from_address(wallet_address: &str, network: &Network) -> Result { let private_key = load_private_key(wallet_address)?; - let wallet = - Wallet::new_from_private_key(network, &private_key).expect("Could not initialize wallet"); + let wallet = Wallet::new_from_private_key(network.clone(), &private_key) + .map_err(|e| eyre!("Could not initialize wallet: {e}"))?; Ok(wallet) } -pub(crate) fn select_wallet() -> Result { - // try if there is a wallet set in the ENV first - if let Ok(env_wallet) = load_evm_wallet_from_env() { - return Ok(env_wallet); - } - - let wallet_address = select_wallet_address()?; - load_wallet_from_address(&wallet_address) +pub(crate) fn select_wallet_from_disk(network: &Network) -> Result { + let wallet_address = select_local_wallet_address()?; + load_wallet_from_address(&wallet_address, network) } -pub(crate) fn select_wallet_private_key() -> Result { - let wallet_address = select_wallet_address()?; +pub(crate) fn select_wallet_private_key() -> Result { + let wallet_address = select_local_wallet_address()?; load_private_key(&wallet_address) } -pub(crate) fn select_wallet_address() -> Result { +pub(crate) fn select_local_wallet_address() -> Result { // Try if a wallet address was already selected this session if let Some(wallet_address) = SELECTED_WALLET_ADDRESS.get() { return Ok(wallet_address.clone()); @@ -142,7 +136,7 @@ pub(crate) fn select_wallet_address() -> Result { let wallet_files = get_wallet_files(&wallets_folder)?; let wallet_address = match wallet_files.len() { - 0 => Err(Error::NoWalletsFound), + 0 => bail!("No local wallets found."), 1 => Ok(filter_wallet_file_extension(&wallet_files[0])), _ => get_wallet_selection(wallet_files), }?; @@ -152,15 +146,15 @@ pub(crate) fn select_wallet_address() -> Result { .to_string()) } -fn get_wallet_selection(wallet_files: Vec) -> Result { +fn get_wallet_selection(wallet_files: Vec) -> Result { list_wallets(&wallet_files); let selected_index = get_wallet_selection_input("Select by index:") .parse::() - .map_err(|_| Error::InvalidSelection)?; + .map_err(|_| eyre!("Invalid wallet selection input"))?; if selected_index < 1 || selected_index > wallet_files.len() { - return Err(Error::InvalidSelection); + bail!("Invalid wallet selection input"); } Ok(filter_wallet_file_extension( @@ -192,9 +186,9 @@ fn list_wallets(wallet_files: &[String]) { table.printstd(); } -fn get_wallet_files(wallets_folder: &PathBuf) -> Result, Error> { +fn get_wallet_files(wallets_folder: &PathBuf) -> Result> { let wallet_files = std::fs::read_dir(wallets_folder) - .map_err(|_| Error::WalletsFolderNotFound)? + .map_err(|e| eyre!("Failed to read wallets folder: {e}"))? .filter_map(Result::ok) .filter_map(|dir_entry| dir_entry.file_name().into_string().ok()) .filter(|file_name| { diff --git a/ant-cli/src/wallet/mod.rs b/ant-cli/src/wallet/mod.rs index ae95594a1b..b24ecf9d97 100644 --- a/ant-cli/src/wallet/mod.rs +++ b/ant-cli/src/wallet/mod.rs @@ -7,24 +7,23 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::keys::{get_secret_key_from_env, load_evm_wallet_from_env}; -use crate::wallet::fs::{select_wallet, select_wallet_private_key}; +use crate::wallet::fs::{select_wallet_from_disk, select_wallet_private_key}; use autonomi::{Network, Wallet}; pub(crate) mod encryption; -pub(crate) mod error; pub(crate) mod fs; pub(crate) mod input; pub const DUMMY_NETWORK: Network = Network::ArbitrumSepolia; /// Load wallet from ENV or disk -pub(crate) fn load_wallet() -> color_eyre::Result { +pub(crate) fn load_wallet(evm_network: &Network) -> color_eyre::Result { // First try wallet from ENV - if let Ok(wallet) = load_evm_wallet_from_env() { + if let Ok(wallet) = load_evm_wallet_from_env(evm_network) { return Ok(wallet); } - let wallet = select_wallet()?; + let wallet = select_wallet_from_disk(evm_network)?; Ok(wallet) } diff --git a/ant-evm/Cargo.toml b/ant-evm/Cargo.toml index 02663b1484..1e15b8d6f8 100644 --- a/ant-evm/Cargo.toml +++ b/ant-evm/Cargo.toml @@ -7,19 +7,18 @@ license = "GPL-3.0" name = "ant-evm" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.8" +version = "0.1.9" [features] -local = ["evmlib/local"] external-signer = ["evmlib/external-signer"] test-utils = [] [dependencies] custom_debug = "~0.6.1" -evmlib = { path = "../evmlib", version = "0.1.8" } +evmlib = { path = "../evmlib", version = "0.1.9" } hex = "~0.4.3" -lazy_static = "~1.4.0" -libp2p = { version = "0.54.1", features = ["identify", "kad"] } +lazy_static = "1.4.0" +libp2p = { version = "0.55.0", features = ["identify", "kad"] } rand = { version = "~0.8.5", features = ["small_rng"] } ring = "0.17.8" rmp-serde = "1.1.1" @@ -34,8 +33,5 @@ xor_name = "5.0.0" [dev-dependencies] tokio = { version = "1.32.0", features = ["macros", "rt"] } -[target.'cfg(target_arch = "wasm32")'.dependencies] -wasmtimer = { version = "0.2.0", features = ["serde"] } - [lints] workspace = true diff --git a/ant-evm/src/amount.rs b/ant-evm/src/amount.rs index be25546042..902fbd1c27 100644 --- a/ant-evm/src/amount.rs +++ b/ant-evm/src/amount.rs @@ -96,6 +96,11 @@ impl FromStr for AttoTokens { EvmError::FailedToParseAttoToken("Can't parse token units".to_string()) })?; + // Check if the units part is too large before multiplication + if units > Amount::from(u64::MAX) { + return Err(EvmError::ExcessiveValue); + } + units .checked_mul(Amount::from(TOKEN_TO_RAW_CONVERSION)) .ok_or(EvmError::ExcessiveValue)? @@ -114,6 +119,9 @@ impl FromStr for AttoTokens { let remainder_conversion = TOKEN_TO_RAW_POWER_OF_10_CONVERSION .checked_sub(remainder_str.len() as u64) .ok_or(EvmError::LossOfPrecision)?; + if remainder_conversion > 32 { + return Err(EvmError::LossOfPrecision); + } parsed_remainder * Amount::from(10).pow(Amount::from(remainder_conversion)) } }; @@ -126,7 +134,7 @@ impl Display for AttoTokens { fn fmt(&self, formatter: &mut Formatter) -> fmt::Result { let unit = self.0 / Amount::from(TOKEN_TO_RAW_CONVERSION); let remainder = self.0 % Amount::from(TOKEN_TO_RAW_CONVERSION); - write!(formatter, "{unit}.{remainder:09}") + write!(formatter, "{unit}.{remainder:032}") } } @@ -160,7 +168,7 @@ mod tests { AttoTokens::from_str("1.000000000000000001")? ); assert_eq!( - AttoTokens::from_u64(1_100_000_000), + AttoTokens::from_u64(1_100_000_000_000_000_000), AttoTokens::from_str("1.1")? ); assert_eq!( @@ -168,16 +176,20 @@ mod tests { AttoTokens::from_str("1.100000000000000001")? ); assert_eq!( - AttoTokens::from_u128(4_294_967_295_000_000_000_000_000_000u128), - AttoTokens::from_str("4294967295")? + AttoTokens::from_u128(4_294_967_295_000_000_000_000_000u128), + AttoTokens::from_str("4294967.295")? + ); + assert_eq!( + AttoTokens::from_u128(4_294_967_295_999_999_999_000_000u128), + AttoTokens::from_str("4294967.295999999999")?, ); assert_eq!( - AttoTokens::from_u128(4_294_967_295_999_999_999_000_000_000_000_000u128), - AttoTokens::from_str("4294967295.999999999")?, + AttoTokens::from_u128(4_294_967_295_999_999_999_000_000u128), + AttoTokens::from_str("4294967.2959999999990000")?, ); assert_eq!( - AttoTokens::from_u128(4_294_967_295_999_999_999_000_000_000_000_000u128), - AttoTokens::from_str("4294967295.9999999990000")?, + AttoTokens::from_u128(18_446_744_074_000_000_000_000_000_000u128), + AttoTokens::from_str("18446744074")? ); assert_eq!( @@ -200,30 +212,39 @@ mod tests { ); assert_eq!( Err(EvmError::LossOfPrecision), - AttoTokens::from_str("0.0000000009") + AttoTokens::from_str("0.0000000000000000001") ); assert_eq!( Err(EvmError::ExcessiveValue), - AttoTokens::from_str("18446744074") + AttoTokens::from_str("340282366920938463463374607431768211455") ); Ok(()) } #[test] fn display() { - assert_eq!("0.000000000", format!("{}", AttoTokens::from_u64(0))); - assert_eq!("0.000000001", format!("{}", AttoTokens::from_u64(1))); - assert_eq!("0.000000010", format!("{}", AttoTokens::from_u64(10))); assert_eq!( - "1.000000000", + "0.00000000000000000000000000000000", + format!("{}", AttoTokens::from_u64(0)) + ); + assert_eq!( + "0.00000000000000000000000000000001", + format!("{}", AttoTokens::from_u64(1)) + ); + assert_eq!( + "0.00000000000000000000000000000010", + format!("{}", AttoTokens::from_u64(10)) + ); + assert_eq!( + "1.00000000000000000000000000000000", format!("{}", AttoTokens::from_u64(1_000_000_000_000_000_000)) ); assert_eq!( - "1.000000001", + "1.00000000000000000000000000000001", format!("{}", AttoTokens::from_u64(1_000_000_000_000_000_001)) ); assert_eq!( - "4294967295.000000000", + "4.00000000000000294967295000000000", format!("{}", AttoTokens::from_u64(4_294_967_295_000_000_000)) ); } @@ -235,11 +256,11 @@ mod tests { AttoTokens::from_u64(1).checked_add(AttoTokens::from_u64(2)) ); assert_eq!( - None, + Some(AttoTokens::from_u128(u64::MAX as u128 + 1)), AttoTokens::from_u64(u64::MAX).checked_add(AttoTokens::from_u64(1)) ); assert_eq!( - None, + Some(AttoTokens::from_u128(u64::MAX as u128 * 2)), AttoTokens::from_u64(u64::MAX).checked_add(AttoTokens::from_u64(u64::MAX)) ); @@ -249,11 +270,7 @@ mod tests { ); assert_eq!( None, - AttoTokens::from_u64(0).checked_sub(AttoTokens::from_u64(u64::MAX)) - ); - assert_eq!( - None, - AttoTokens::from_u64(10).checked_sub(AttoTokens::from_u64(11)) + AttoTokens::from_u64(0).checked_sub(AttoTokens::from_u64(1)) ); } } diff --git a/ant-evm/src/data_payments.rs b/ant-evm/src/data_payments.rs index 48f904f8d4..f8154f44e1 100644 --- a/ant-evm/src/data_payments.rs +++ b/ant-evm/src/data_payments.rs @@ -10,14 +10,10 @@ use crate::EvmError; use evmlib::{ common::{Address as RewardsAddress, QuoteHash}, quoting_metrics::QuotingMetrics, - utils::dummy_address, }; use libp2p::{identity::PublicKey, PeerId}; use serde::{Deserialize, Serialize}; -#[cfg(not(target_arch = "wasm32"))] pub use std::time::SystemTime; -#[cfg(target_arch = "wasm32")] -pub use wasmtimer::std::SystemTime; use xor_name::XorName; /// The time in seconds that a quote is valid for @@ -92,6 +88,9 @@ impl ProofOfPayment { pub fn verify_for(&self, peer_id: PeerId) -> bool { // make sure I am in the list of payees if !self.payees().contains(&peer_id) { + warn!("Payment does not contain node peer id"); + debug!("Payment contains peer ids: {:?}", self.payees()); + debug!("Node peer id: {:?}", peer_id); return false; } @@ -105,11 +104,23 @@ impl ProofOfPayment { } }; if !quote.check_is_signed_by_claimed_peer(peer_id) { + warn!("Payment is not signed by claimed peer"); return false; } } true } + + /// Verifies whether all quotes were made for the expected data type. + pub fn verify_data_type(&self, data_type: u32) -> bool { + for (_, quote) in self.peer_quotes.iter() { + if quote.quoting_metrics.data_type != data_type { + return false; + } + } + + true + } } /// A payment quote to store data given by a node to a client @@ -134,18 +145,6 @@ pub struct PaymentQuote { } impl PaymentQuote { - /// create an empty PaymentQuote - pub fn zero() -> Self { - Self { - content: Default::default(), - timestamp: SystemTime::now(), - quoting_metrics: Default::default(), - rewards_address: dummy_address(), - pub_key: vec![], - signature: vec![], - } - } - pub fn hash(&self) -> QuoteHash { let mut bytes = self.bytes_for_sig(); bytes.extend_from_slice(self.pub_key.as_slice()); @@ -189,7 +188,7 @@ impl PaymentQuote { if let Ok(pub_key) = libp2p::identity::PublicKey::try_decode_protobuf(&self.pub_key) { Ok(PeerId::from(pub_key.clone())) } else { - error!("Cann't parse PublicKey from protobuf"); + error!("Can't parse PublicKey from protobuf"); Err(EvmError::InvalidQuotePublicKey) } } @@ -199,7 +198,7 @@ impl PaymentQuote { let pub_key = if let Ok(pub_key) = PublicKey::try_decode_protobuf(&self.pub_key) { pub_key } else { - error!("Cann't parse PublicKey from protobuf"); + error!("Can't parse PublicKey from protobuf"); return false; }; @@ -232,11 +231,24 @@ impl PaymentQuote { } /// test utility to create a dummy quote + #[cfg(test)] pub fn test_dummy(xorname: XorName) -> Self { + use evmlib::utils::dummy_address; + Self { content: xorname, timestamp: SystemTime::now(), - quoting_metrics: Default::default(), + quoting_metrics: QuotingMetrics { + data_size: 0, + data_type: 0, + close_records_stored: 0, + records_per_type: vec![], + max_records: 0, + received_payment_count: 0, + live_time: 0, + network_density: None, + network_size: None, + }, pub_key: vec![], signature: vec![], rewards_address: dummy_address(), @@ -328,9 +340,9 @@ mod tests { #[test] fn test_is_newer_than() { - let old_quote = PaymentQuote::zero(); + let old_quote = PaymentQuote::test_dummy(Default::default()); sleep(Duration::from_millis(100)); - let new_quote = PaymentQuote::zero(); + let new_quote = PaymentQuote::test_dummy(Default::default()); assert!(new_quote.is_newer_than(&old_quote)); assert!(!old_quote.is_newer_than(&new_quote)); } @@ -342,7 +354,7 @@ mod tests { let false_peer = PeerId::random(); - let mut quote = PaymentQuote::zero(); + let mut quote = PaymentQuote::test_dummy(Default::default()); let bytes = quote.bytes_for_sig(); let signature = if let Ok(sig) = keypair.sign(&bytes) { sig @@ -373,9 +385,9 @@ mod tests { #[test] fn test_historical_verify() { - let mut old_quote = PaymentQuote::zero(); + let mut old_quote = PaymentQuote::test_dummy(Default::default()); sleep(Duration::from_millis(100)); - let mut new_quote = PaymentQuote::zero(); + let mut new_quote = PaymentQuote::test_dummy(Default::default()); // historical_verify will swap quotes to compare based on timeline automatically assert!(new_quote.historical_verify(&old_quote)); diff --git a/ant-evm/src/lib.rs b/ant-evm/src/lib.rs index ece2c36083..e8d5e92784 100644 --- a/ant-evm/src/lib.rs +++ b/ant-evm/src/lib.rs @@ -19,7 +19,7 @@ pub use evmlib::cryptography; #[cfg(feature = "external-signer")] pub use evmlib::external_signer; pub use evmlib::utils; -pub use evmlib::utils::get_evm_network_from_env; +pub use evmlib::utils::get_evm_network; pub use evmlib::utils::{DATA_PAYMENTS_ADDRESS, PAYMENT_TOKEN_ADDRESS, RPC_URL}; pub use evmlib::wallet::Error as EvmWalletError; pub use evmlib::wallet::Wallet as EvmWallet; diff --git a/ant-logging/Cargo.toml b/ant-logging/Cargo.toml index af720fbc7f..a27bec98ee 100644 --- a/ant-logging/Cargo.toml +++ b/ant-logging/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-logging" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.2.45" +version = "0.2.46" [dependencies] chrono = "~0.4.19" @@ -29,7 +29,7 @@ tracing-opentelemetry = { version = "0.21", optional = true } tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] -color-eyre = "~0.6" +color-eyre = "0.6.3" tracing-test = "0.2.4" [features] diff --git a/ant-logging/src/layers.rs b/ant-logging/src/layers.rs index 3e2d55d490..5886f56826 100644 --- a/ant-logging/src/layers.rs +++ b/ant-logging/src/layers.rs @@ -281,7 +281,6 @@ fn get_logging_targets(logging_env_value: &str) -> Result> ("ant_node_manager".to_string(), Level::TRACE), ("ant_node_rpc_client".to_string(), Level::TRACE), ("ant_protocol".to_string(), Level::TRACE), - ("ant_registers".to_string(), Level::INFO), ("ant_service_management".to_string(), Level::TRACE), ("autonomi".to_string(), Level::TRACE), ("evmlib".to_string(), Level::TRACE), diff --git a/ant-metrics/Cargo.toml b/ant-metrics/Cargo.toml index 42734a64ae..fb9f77b744 100644 --- a/ant-metrics/Cargo.toml +++ b/ant-metrics/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-metrics" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.24" +version = "0.1.25" [[bin]] path = "src/main.rs" @@ -15,7 +15,7 @@ name = "metrics" [dependencies] clap = { version = "4.2.1", features = ["cargo", "string"] } -color-eyre = "~0.6.2" +color-eyre = "0.6.3" dirs-next = "~2.0.0" regex = "1.10" serde = { version = "1.0.133", features = ["derive"] } diff --git a/ant-metrics/grafana/provisioning/dashboards/safe-network.json b/ant-metrics/grafana/provisioning/dashboards/safe-network.json index cdbc296e1b..7b56d33d43 100644 --- a/ant-metrics/grafana/provisioning/dashboards/safe-network.json +++ b/ant-metrics/grafana/provisioning/dashboards/safe-network.json @@ -369,204 +369,6 @@ "title": "Chunk PUTs", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasourceuuid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.1.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasourceuuid" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sn_node_put_record_ok_total{node_id=~\"$var_node_list\", record_type=\"Spend\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{node_id}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Spend PUTs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasourceuuid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.1.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasourceuuid" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sn_node_put_record_ok_total{node_id=~\"$var_node_list\", record_type=\"Register\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{node_id}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Register PUTs", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", diff --git a/ant-networking/Cargo.toml b/ant-networking/Cargo.toml index ecfcdc4462..e955fce386 100644 --- a/ant-networking/Cargo.toml +++ b/ant-networking/Cargo.toml @@ -7,25 +7,21 @@ license = "GPL-3.0" name = "ant-networking" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.3.4" +version = "0.3.5" [features] default = [] -encrypt-records = [] -local = ["libp2p/mdns"] loud = [] open-metrics = ["libp2p/metrics", "prometheus-client", "hyper", "sysinfo"] -# tcp is automatically enabled when compiling for wasm32 -upnp = ["libp2p/upnp"] [dependencies] aes-gcm-siv = "0.11.1" -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } -ant-registers = { path = "../ant-registers", version = "0.4.7" } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } async-trait = "0.1" +bls = { package = "blsttc", version = "8.0.2" } bytes = { version = "1.0.1", features = ["serde"] } custom_debug = "~0.6.1" exponential-backoff = "2.0.0" @@ -38,10 +34,10 @@ hyper = { version = "0.14", features = [ "http1", ], optional = true } itertools = "~0.12.1" -lazy_static = "~1.4.0" -libp2p = { version = "0.54.1", features = [ +libp2p = { version = "0.55.0", features = [ "tokio", "dns", + "upnp", "kad", "macros", "request-response", @@ -58,13 +54,11 @@ prometheus-client = { version = "0.22", optional = true } rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" rmp-serde = "1.1.1" -self_encryption = "~0.30.0" serde = { version = "1.0.133", features = ["derive", "rc"] } sha2 = "0.10" strum = { version = "0.26.2", features = ["derive"] } sysinfo = { version = "0.30.8", default-features = false, optional = true } thiserror = "1.0.23" -tiny-keccak = { version = "~2.0.2", features = ["sha3"] } tokio = { version = "1.32.0", features = [ "io-util", "macros", @@ -79,7 +73,6 @@ xor_name = "5.0.0" [dev-dependencies] assert_fs = "1.0.0" -bls = { package = "blsttc", version = "8.0.1" } eyre = "0.6.8" # add rand to libp2p libp2p-identity = { version = "0.2.7", features = ["rand"] } @@ -89,25 +82,5 @@ uuid = { version = "1.5.0", features = ["v4"] } [lints] workspace = true -# wasm build requirements [lib] crate-type = ["cdylib", "rlib"] - -[target.'cfg(target_arch = "wasm32")'.dependencies] -getrandom = { version = "0.2.12", features = ["js"] } -libp2p = { version = "0.54.1", features = [ - "tokio", - "dns", - "kad", - "tcp", - "macros", - "request-response", - "cbor", - "identify", - "noise", - "yamux", - "websocket-websys", - "wasm-bindgen", -] } -wasmtimer = "0.2.0" -wasm-bindgen-futures = "0.4.40" diff --git a/ant-networking/src/bootstrap.rs b/ant-networking/src/bootstrap.rs index c03fa52a61..30511009c9 100644 --- a/ant-networking/src/bootstrap.rs +++ b/ant-networking/src/bootstrap.rs @@ -11,7 +11,7 @@ use libp2p::kad::K_VALUE; use rand::{rngs::OsRng, Rng}; use tokio::time::Duration; -use crate::target_arch::{interval, Instant, Interval}; +use crate::time::{interval, Instant, Interval}; /// The default interval at which NetworkDiscovery is triggered. /// The interval is increased as more peers are added to the routing table. @@ -53,19 +53,22 @@ impl SwarmDriver { let now = Instant::now(); // Find the farthest bucket that is not full. This is used to skip refreshing the RT of farthest full buckets. - let mut farthest_unfilled_bucket = 0; + let mut first_filled_bucket = 0; + // unfilled kbuckets will not be returned, hence the value shall be: + // * first_filled_kbucket.ilog2() - 1 for kbucket in self.swarm.behaviour_mut().kademlia.kbuckets() { let Some(ilog2) = kbucket.range().0.ilog2() else { continue; }; - if kbucket.num_entries() < K_VALUE.get() && ilog2 > farthest_unfilled_bucket { - farthest_unfilled_bucket = ilog2; + if kbucket.num_entries() >= K_VALUE.get() { + first_filled_bucket = ilog2; + break; } } - let farthest_unfilled_bucket = if farthest_unfilled_bucket == 0 { + let farthest_unfilled_bucket = if first_filled_bucket == 0 { None } else { - Some(farthest_unfilled_bucket) + Some(first_filled_bucket - 1) }; let addrs = self.network_discovery.candidates(farthest_unfilled_bucket); @@ -131,7 +134,6 @@ impl ContinuousNetworkDiscover { /// Returns `true` if we should carry out the Kademlia Bootstrap process immediately. /// Also optionally returns the new interval for network discovery. - #[cfg_attr(target_arch = "wasm32", allow(clippy::unused_async))] pub(crate) async fn should_we_discover( &self, peers_in_rt: u32, @@ -161,10 +163,7 @@ impl ContinuousNetworkDiscover { "It has been {LAST_PEER_ADDED_TIME_LIMIT:?} since we last added a peer to RT. Slowing down the continuous network discovery process. Old interval: {current_interval:?}, New interval: {no_peer_added_slowdown_interval_duration:?}" ); - // `Interval` ticks immediately for Tokio, but not for `wasmtimer`, which is used for wasm32. - #[cfg_attr(target_arch = "wasm32", allow(unused_mut))] let mut new_interval = interval(no_peer_added_slowdown_interval_duration); - #[cfg(not(target_arch = "wasm32"))] new_interval.tick().await; return (should_network_discover, Some(new_interval)); @@ -177,10 +176,7 @@ impl ContinuousNetworkDiscover { let new_interval = if new_interval > current_interval { info!("More peers have been added to our RT!. Slowing down the continuous network discovery process. Old interval: {current_interval:?}, New interval: {new_interval:?}"); - // `Interval` ticks immediately for Tokio, but not for `wasmtimer`, which is used for wasm32. - #[cfg_attr(target_arch = "wasm32", allow(unused_mut))] let mut interval = interval(new_interval); - #[cfg(not(target_arch = "wasm32"))] interval.tick().await; Some(interval) diff --git a/ant-networking/src/cmd.rs b/ant-networking/src/cmd.rs index 9a694f0650..bc42fa4fbf 100644 --- a/ant-networking/src/cmd.rs +++ b/ant-networking/src/cmd.rs @@ -7,23 +7,24 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ + config::GetRecordCfg, driver::{PendingGetClosestType, SwarmDriver}, error::{NetworkError, Result}, event::TerminateNodeReason, log_markers::Marker, - multiaddr_pop_p2p, GetRecordCfg, GetRecordError, MsgResponder, NetworkEvent, CLOSE_GROUP_SIZE, + multiaddr_pop_p2p, GetRecordError, MsgResponder, NetworkEvent, ResponseQuorum, + CLOSE_GROUP_SIZE, }; -use ant_evm::{PaymentQuote, QuotingMetrics, U256}; +use ant_evm::{PaymentQuote, QuotingMetrics}; use ant_protocol::{ - convert_distance_to_u256, messages::{Cmd, Request, Response}, - storage::{RecordHeader, RecordKind, RecordType}, + storage::{DataTypes, RecordHeader, RecordKind, ValidationType}, NetworkAddress, PrettyPrintRecordKey, }; use libp2p::{ kad::{ store::{Error as StoreError, RecordStore}, - KBucketDistance as Distance, Quorum, Record, RecordKey, + KBucketDistance as Distance, Record, RecordKey, K_VALUE, }, Multiaddr, PeerId, }; @@ -35,7 +36,7 @@ use std::{ use tokio::sync::oneshot; use xor_name::XorName; -use crate::target_arch::Instant; +use crate::time::Instant; const MAX_CONTINUOUS_HDD_WRITE_ERROR: usize = 5; @@ -45,8 +46,10 @@ const REPLICATION_TIMEOUT: Duration = Duration::from_secs(45); // Throttles replication to at most once every 30 seconds const MIN_REPLICATION_INTERVAL_S: Duration = Duration::from_secs(30); -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq, Clone)] pub enum NodeIssue { + /// Some connections might be considered to be critical and should be tracked. + ConnectionIssue, /// Data Replication failed ReplicationFailure, /// Close nodes have reported this peer as bad @@ -57,6 +60,18 @@ pub enum NodeIssue { FailedChunkProofCheck, } +impl std::fmt::Display for NodeIssue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + NodeIssue::ConnectionIssue => write!(f, "CriticalConnectionIssue"), + NodeIssue::ReplicationFailure => write!(f, "ReplicationFailure"), + NodeIssue::CloseNodesShunning => write!(f, "CloseNodesShunning"), + NodeIssue::BadQuoting => write!(f, "BadQuoting"), + NodeIssue::FailedChunkProofCheck => write!(f, "FailedChunkProofCheck"), + } + } +} + /// Commands to send to the Swarm pub enum LocalSwarmCmd { /// Get a list of all peers in local RT, with correspondent Multiaddr info attached as well. @@ -72,7 +87,7 @@ pub enum LocalSwarmCmd { /// In case the range is too narrow, returns at lease CLOSE_GROUP_SIZE peers. GetReplicateCandidates { data_addr: NetworkAddress, - sender: oneshot::Sender>, + sender: oneshot::Sender>>, }, // Returns up to K_VALUE peers from all the k-buckets from the local Routing Table. // And our PeerId as well. @@ -88,11 +103,11 @@ pub enum LocalSwarmCmd { /// Check if the local RecordStore contains the provided key RecordStoreHasKey { key: RecordKey, - sender: oneshot::Sender, + sender: oneshot::Sender>, }, /// Get the Addresses of all the Records held locally GetAllLocalRecordAddresses { - sender: oneshot::Sender>, + sender: oneshot::Sender>>, }, /// Get data from the local RecordStore GetLocalRecord { @@ -103,13 +118,16 @@ pub enum LocalSwarmCmd { /// Returns the quoting metrics and whether the record at `key` is already stored locally GetLocalQuotingMetrics { key: RecordKey, - sender: oneshot::Sender<(QuotingMetrics, bool)>, + data_type: u32, + data_size: usize, + sender: oneshot::Sender>, }, /// Notify the node received a payment. PaymentReceived, /// Put record to the local RecordStore PutLocalRecord { record: Record, + is_client_put: bool, }, /// Remove a local record from the RecordStore /// Typically because the write failed @@ -120,7 +138,8 @@ pub enum LocalSwarmCmd { /// This should be done after the record has been stored to disk AddLocalRecordAsStored { key: RecordKey, - record_type: RecordType, + record_type: ValidationType, + data_type: DataTypes, }, /// Add a peer to the blocklist AddPeerToBlockList { @@ -141,7 +160,7 @@ pub enum LocalSwarmCmd { quotes: Vec<(PeerId, PaymentQuote)>, }, // Notify a fetch completion - FetchCompleted((RecordKey, RecordType)), + FetchCompleted((RecordKey, ValidationType)), /// Triggers interval repliation /// NOTE: This does result in outgoing messages, but is produced locally TriggerIntervalReplication, @@ -151,6 +170,15 @@ pub enum LocalSwarmCmd { AddNetworkDensitySample { distance: Distance, }, + /// Send peer scores (collected from storage challenge) to replication_fetcher + NotifyPeerScores { + peer_scores: Vec<(PeerId, bool)>, + }, + /// Add fresh replicate records into replication_fetcher + AddFreshReplicateRecords { + holder: NetworkAddress, + keys: Vec<(NetworkAddress, ValidationType)>, + }, } /// Commands to send to the Swarm @@ -194,14 +222,14 @@ pub enum NetworkSwarmCmd { PutRecord { record: Record, sender: oneshot::Sender>, - quorum: Quorum, + quorum: ResponseQuorum, }, /// Put record to specific node PutRecordTo { peers: Vec, record: Record, sender: oneshot::Sender>, - quorum: Quorum, + quorum: ResponseQuorum, }, } @@ -210,10 +238,13 @@ pub enum NetworkSwarmCmd { impl Debug for LocalSwarmCmd { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - LocalSwarmCmd::PutLocalRecord { record } => { + LocalSwarmCmd::PutLocalRecord { + record, + is_client_put, + } => { write!( f, - "LocalSwarmCmd::PutLocalRecord {{ key: {:?} }}", + "LocalSwarmCmd::PutLocalRecord {{ key: {:?}, is_client_put: {is_client_put:?} }}", PrettyPrintRecordKey::from(&record.key) ) } @@ -224,10 +255,14 @@ impl Debug for LocalSwarmCmd { PrettyPrintRecordKey::from(key) ) } - LocalSwarmCmd::AddLocalRecordAsStored { key, record_type } => { + LocalSwarmCmd::AddLocalRecordAsStored { + key, + record_type, + data_type, + } => { write!( f, - "LocalSwarmCmd::AddLocalRecordAsStored {{ key: {:?}, record_type: {record_type:?} }}", + "LocalSwarmCmd::AddLocalRecordAsStored {{ key: {:?}, record_type: {record_type:?}, data_type: {data_type:?} }}", PrettyPrintRecordKey::from(key) ) } @@ -310,6 +345,15 @@ impl Debug for LocalSwarmCmd { LocalSwarmCmd::AddNetworkDensitySample { distance } => { write!(f, "LocalSwarmCmd::AddNetworkDensitySample({distance:?})") } + LocalSwarmCmd::NotifyPeerScores { peer_scores } => { + write!(f, "LocalSwarmCmd::NotifyPeerScores({peer_scores:?})") + } + LocalSwarmCmd::AddFreshReplicateRecords { holder, keys } => { + write!( + f, + "LocalSwarmCmd::AddFreshReplicateRecords({holder:?}, {keys:?})" + ) + } } } } @@ -431,7 +475,7 @@ impl SwarmDriver { .swarm .behaviour_mut() .kademlia - .put_record(record, quorum) + .put_record(record, quorum.get_kad_quorum()) { Ok(request_id) => { debug!("Sent record {record_key:?} to network. Request id: {request_id:?} to network"); @@ -463,7 +507,7 @@ impl SwarmDriver { let request_id = self.swarm.behaviour_mut().kademlia.put_record_to( record, peers.into_iter(), - quorum, + quorum.get_kad_quorum(), ); debug!("Sent record {record_key:?} to {peers_count:?} peers. Request id: {request_id:?}"); @@ -575,7 +619,12 @@ impl SwarmDriver { cmd_string = "TriggerIntervalReplication"; self.try_interval_replication()?; } - LocalSwarmCmd::GetLocalQuotingMetrics { key, sender } => { + LocalSwarmCmd::GetLocalQuotingMetrics { + key, + data_type, + data_size, + sender, + } => { cmd_string = "GetLocalQuotingMetrics"; let ( _index, @@ -586,13 +635,23 @@ impl SwarmDriver { ) = self.kbuckets_status(); let estimated_network_size = Self::estimate_network_size(peers_in_non_full_buckets, num_of_full_buckets); - let (quoting_metrics, is_already_stored) = self + let (quoting_metrics, is_already_stored) = match self .swarm .behaviour_mut() .kademlia .store_mut() - .quoting_metrics(&key, Some(estimated_network_size as u64)); - + .quoting_metrics( + &key, + data_type, + data_size, + Some(estimated_network_size as u64), + ) { + Ok(res) => res, + Err(err) => { + let _res = sender.send(Err(err)); + return Ok(()); + } + }; self.record_metrics(Marker::QuotingMetrics { quoting_metrics: "ing_metrics, }); @@ -631,7 +690,7 @@ impl SwarmDriver { .retain(|peer_addr| key_address.distance(peer_addr) < boundary_distance); } - let _res = sender.send((quoting_metrics, is_already_stored)); + let _res = sender.send(Ok((quoting_metrics, is_already_stored))); } LocalSwarmCmd::PaymentReceived => { cmd_string = "PaymentReceived"; @@ -653,7 +712,10 @@ impl SwarmDriver { let _ = sender.send(record); } - LocalSwarmCmd::PutLocalRecord { record } => { + LocalSwarmCmd::PutLocalRecord { + record, + is_client_put, + } => { cmd_string = "PutLocalRecord"; let key = record.key.clone(); let record_key = PrettyPrintRecordKey::from(&key); @@ -661,16 +723,12 @@ impl SwarmDriver { let record_type = match RecordHeader::from_record(&record) { Ok(record_header) => { match record_header.kind { - RecordKind::Chunk => RecordType::Chunk, - RecordKind::Scratchpad => RecordType::Scratchpad, - RecordKind::Transaction | RecordKind::Register => { + RecordKind::DataOnly(DataTypes::Chunk) => ValidationType::Chunk, + RecordKind::DataOnly(_) => { let content_hash = XorName::from_content(&record.value); - RecordType::NonChunk(content_hash) + ValidationType::NonChunk(content_hash) } - RecordKind::ChunkWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::TransactionWithPayment - | RecordKind::ScratchpadWithPayment => { + RecordKind::DataWithPayment(_) => { error!("Record {record_key:?} with payment shall not be stored locally."); return Err(NetworkError::InCorrectRecordHeader); } @@ -687,7 +745,7 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .put_verified(record, record_type.clone()); + .put_verified(record, record_type.clone(), is_client_put); match result { Ok(_) => { @@ -706,7 +764,7 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .get_farthest(); + .get_farthest()?; self.replication_fetcher.set_farthest_on_full(farthest); } Err(_) => { @@ -734,7 +792,7 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .get_farthest_replication_distance() + .get_farthest_replication_distance()? { self.replication_fetcher .set_replication_distance_range(distance); @@ -747,17 +805,17 @@ impl SwarmDriver { return Err(err.into()); }; } - LocalSwarmCmd::AddLocalRecordAsStored { key, record_type } => { - info!( - "Adding Record locally, for {:?} and {record_type:?}", - PrettyPrintRecordKey::from(&key) - ); + LocalSwarmCmd::AddLocalRecordAsStored { + key, + record_type, + data_type, + } => { cmd_string = "AddLocalRecordAsStored"; self.swarm .behaviour_mut() .kademlia .store_mut() - .mark_as_stored(key, record_type); + .mark_as_stored(key, record_type, data_type); // Reset counter on any success HDD write. self.hard_disk_write_error = 0; } @@ -924,6 +982,14 @@ impl SwarmDriver { cmd_string = "AddNetworkDensitySample"; self.network_density_samples.add(distance); } + LocalSwarmCmd::NotifyPeerScores { peer_scores } => { + cmd_string = "NotifyPeerScores"; + self.replication_fetcher.add_peer_scores(peer_scores); + } + LocalSwarmCmd::AddFreshReplicateRecords { holder, keys } => { + cmd_string = "AddFreshReplicateRecords"; + let _ = self.add_keys_to_replication_fetcher(holder, keys, true); + } } self.log_handling(cmd_string.to_string(), start.elapsed()); @@ -931,12 +997,11 @@ impl SwarmDriver { Ok(()) } - fn record_node_issue(&mut self, peer_id: PeerId, issue: NodeIssue) { + pub(crate) fn record_node_issue(&mut self, peer_id: PeerId, issue: NodeIssue) { info!("Peer {peer_id:?} is reported as having issue {issue:?}"); let (issue_vec, is_bad) = self.bad_nodes.entry(peer_id).or_default(); - - let mut is_new_bad = false; - let mut bad_behaviour: String = "".to_string(); + let mut new_bad_behaviour = None; + let mut is_connection_issue = false; // If being considered as bad already, skip certain operations if !(*is_bad) { @@ -969,9 +1034,13 @@ impl SwarmDriver { .filter(|(i, _timestamp)| *issue == *i) .count(); if issue_counts >= 3 { - *is_bad = true; - is_new_bad = true; - bad_behaviour = format!("{issue:?}"); + // If it is a connection issue, we don't need to consider it as a bad node + if matches!(issue, NodeIssue::ConnectionIssue) { + is_connection_issue = true; + } else { + *is_bad = true; + } + new_bad_behaviour = Some(issue.clone()); info!("Peer {peer_id:?} accumulated {issue_counts} times of issue {issue:?}. Consider it as a bad node now."); // Once a bad behaviour detected, no point to continue break; @@ -979,16 +1048,28 @@ impl SwarmDriver { } } + // Give the faulty connection node more chances by removing the issue from the list. It is still evicted from + // the routing table. + if is_connection_issue { + issue_vec.retain(|(issue, _timestamp)| !matches!(issue, NodeIssue::ConnectionIssue)); + info!("Evicting bad peer {peer_id:?} due to connection issue from RT."); + if let Some(dead_peer) = self.swarm.behaviour_mut().kademlia.remove_peer(&peer_id) { + self.update_on_peer_removal(*dead_peer.node.key.preimage()); + } + return; + } + if *is_bad { - warn!("Cleaning out bad_peer {peer_id:?}. Will be added to the blocklist after informing that peer."); + info!("Evicting bad peer {peer_id:?} from RT."); if let Some(dead_peer) = self.swarm.behaviour_mut().kademlia.remove_peer(&peer_id) { self.update_on_peer_removal(*dead_peer.node.key.preimage()); } - if is_new_bad { + if let Some(bad_behaviour) = new_bad_behaviour { + // inform the bad node about it and add to the blocklist after that (not for connection issues) self.record_metrics(Marker::PeerConsideredAsBad { bad_peer: &peer_id }); - // inform the bad node about it and add to the blocklist after that. + warn!("Peer {peer_id:?} is considered as bad due to {bad_behaviour:?}. Informing the peer and adding to blocklist."); // response handling let (tx, rx) = oneshot::channel(); let local_swarm_cmd_sender = self.local_cmd_sender.clone(); @@ -1013,7 +1094,7 @@ impl SwarmDriver { let request = Request::Cmd(Cmd::PeerConsideredAsBad { detected_by: NetworkAddress::from_peer(self.self_peer_id), bad_peer: NetworkAddress::from_peer(peer_id), - bad_behaviour, + bad_behaviour: bad_behaviour.to_string(), }); self.queue_network_swarm_cmd(NetworkSwarmCmd::SendRequest { req: request, @@ -1052,7 +1133,7 @@ impl SwarmDriver { self.last_replication = Some(Instant::now()); let self_addr = NetworkAddress::from_peer(self.self_peer_id); - let mut replicate_targets = self.get_replicate_candidates(&self_addr); + let mut replicate_targets = self.get_replicate_candidates(&self_addr)?; let now = Instant::now(); self.replication_targets @@ -1068,7 +1149,7 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .record_addresses_ref() + .record_addresses_ref()? .values() .cloned() .collect(); @@ -1080,7 +1161,10 @@ impl SwarmDriver { ); let request = Request::Cmd(Cmd::Replicate { holder: NetworkAddress::from_peer(self.self_peer_id), - keys: all_records, + keys: all_records + .into_iter() + .map(|(addr, val_type, _data_type)| (addr, val_type)) + .collect(), }); for peer_id in replicate_targets { self.queue_network_swarm_cmd(NetworkSwarmCmd::SendRequest { @@ -1099,11 +1183,22 @@ impl SwarmDriver { } // Replies with in-range replicate candidates - // Fall back to CLOSE_GROUP_SIZE peers if range is too narrow. + // Fall back to expected_candidates peers if range is too narrow. // Note that: - // * For general replication, replicate candidates shall be the closest to self + // * For general replication, replicate candidates shall be closest to self, but with wider range // * For replicate fresh records, the replicate candidates shall be the closest to data - pub(crate) fn get_replicate_candidates(&mut self, target: &NetworkAddress) -> Vec { + + pub(crate) fn get_replicate_candidates( + &mut self, + target: &NetworkAddress, + ) -> Result> { + let is_periodic_replicate = target.as_peer_id().is_some(); + let expected_candidates = if is_periodic_replicate { + CLOSE_GROUP_SIZE * 2 + } else { + CLOSE_GROUP_SIZE + }; + // get closest peers from buckets, sorted by increasing distance to the target let kbucket_key = target.as_kbucket_key(); let closest_k_peers: Vec = self @@ -1113,6 +1208,7 @@ impl SwarmDriver { .get_closest_local_peers(&kbucket_key) // Map KBucketKey to PeerId. .map(|key| key.into_preimage()) + .take(K_VALUE.get()) .collect(); if let Some(responsible_range) = self @@ -1120,32 +1216,30 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .get_farthest_replication_distance() + .get_farthest_replication_distance()? { let peers_in_range = get_peers_in_range(&closest_k_peers, target, responsible_range); - if peers_in_range.len() >= CLOSE_GROUP_SIZE { - return peers_in_range; + if peers_in_range.len() >= expected_candidates { + return Ok(peers_in_range); } } - // In case the range is too narrow, fall back to at least CLOSE_GROUP_SIZE peers. - closest_k_peers + // In case the range is too narrow, fall back to at least expected_candidates peers. + Ok(closest_k_peers .iter() - .take(CLOSE_GROUP_SIZE) + .take(expected_candidates) .cloned() - .collect() + .collect()) } } /// Returns the nodes that within the defined distance. -fn get_peers_in_range(peers: &[PeerId], address: &NetworkAddress, range: U256) -> Vec { +fn get_peers_in_range(peers: &[PeerId], address: &NetworkAddress, range: Distance) -> Vec { peers .iter() .filter_map(|peer_id| { - let distance = - convert_distance_to_u256(&address.distance(&NetworkAddress::from_peer(*peer_id))); - if distance <= range { + if address.distance(&NetworkAddress::from_peer(*peer_id)) <= range { Some(*peer_id) } else { None diff --git a/ant-networking/src/config.rs b/ant-networking/src/config.rs new file mode 100644 index 0000000000..36a5c1a334 --- /dev/null +++ b/ant-networking/src/config.rs @@ -0,0 +1,226 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_protocol::{ + messages::{ChunkProof, Nonce}, + PrettyPrintRecordKey, CLOSE_GROUP_SIZE, +}; +use core::fmt::{self, Debug}; +use exponential_backoff::Backoff; +use libp2p::{ + kad::{Quorum, Record}, + PeerId, +}; +use std::{collections::HashSet, num::NonZeroUsize, time::Duration}; + +use crate::close_group_majority; + +/// A strategy that translates into a configuration for exponential backoff. +/// The first retry is done after 2 seconds, after which the backoff is roughly doubled each time. +/// The interval does not go beyond 32 seconds. So the intervals increase from 2 to 4, to 8, to 16, to 32 seconds and +/// all attempts are made at most 32 seconds apart. +/// +/// The exact timings depend on jitter, which is set to 0.2, meaning the intervals can deviate quite a bit +/// from the ones listed in the docs. +/// +/// The default strategy is `Balanced`. +#[derive(Clone, Debug, Copy, Default)] +pub enum RetryStrategy { + /// Attempt once (no retries) + None, + /// Retry 3 times (waits 2s, 4s and lastly 8s; max total time ~14s) + Quick, + /// Retry 5 times (waits 2s, 4s, 8s, 16s and lastly 32s; max total time ~62s) + #[default] + Balanced, + /// Retry 9 times (waits 2s, 4s, 8s, 16s, 32s, 32s, 32s, 32s and lastly 32s; max total time ~190s) + Persistent, + /// Attempt a specific number of times + N(NonZeroUsize), +} + +impl RetryStrategy { + pub fn attempts(&self) -> usize { + match self { + RetryStrategy::None => 1, + RetryStrategy::Quick => 4, + RetryStrategy::Balanced => 6, + RetryStrategy::Persistent => 10, + RetryStrategy::N(x) => x.get(), + } + } + + pub fn backoff(&self) -> Backoff { + let mut backoff = Backoff::new( + self.attempts() as u32, + Duration::from_secs(1), // First interval is double of this (see https://github.com/yoshuawuyts/exponential-backoff/issues/23) + Some(Duration::from_secs(32)), + ); + backoff.set_factor(2); // Default. + backoff.set_jitter(0.2); // Default is 0.3. + backoff + } +} + +impl fmt::Display for RetryStrategy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self:?}") + } +} + +/// Specifies the minimum number of distinct nodes that must be successfully contacted in order for a query to succeed. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ResponseQuorum { + One, + Majority, + All, + N(NonZeroUsize), +} + +impl std::str::FromStr for ResponseQuorum { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "one" => Ok(ResponseQuorum::One), + "majority" => Ok(ResponseQuorum::Majority), + "all" => Ok(ResponseQuorum::All), + _ => { + if let Ok(n) = s.parse::() { + let n = NonZeroUsize::new(n); + match n { + Some(n) => Ok(ResponseQuorum::N(n)), + None => Err("Quorum value must be greater than 0".to_string()), + } + } else { + Err("Invalid quorum value".to_string()) + } + } + } + } +} + +impl ResponseQuorum { + pub(crate) fn get_kad_quorum(&self) -> Quorum { + match self { + ResponseQuorum::One => Quorum::One, + ResponseQuorum::Majority => Quorum::Majority, + ResponseQuorum::All => Quorum::All, + ResponseQuorum::N(n) => Quorum::N(*n), + } + } + + /// Get the value of the provided Quorum + pub fn get_value(&self) -> usize { + match self { + ResponseQuorum::Majority => close_group_majority(), + ResponseQuorum::All => CLOSE_GROUP_SIZE, + ResponseQuorum::N(v) => v.get(), + ResponseQuorum::One => 1, + } + } +} + +/// The various settings to apply to when fetching a record from network +#[derive(Clone)] +pub struct GetRecordCfg { + /// The query will result in an error if we get records less than the provided Quorum + pub get_quorum: ResponseQuorum, + /// If enabled, the provided `RetryStrategy` is used to retry if a GET attempt fails. + pub retry_strategy: RetryStrategy, + /// Only return if we fetch the provided record. + pub target_record: Option, + /// Logs if the record was not fetched from the provided set of peers. + pub expected_holders: HashSet, +} + +impl GetRecordCfg { + pub fn does_target_match(&self, record: &Record) -> bool { + if let Some(ref target_record) = self.target_record { + target_record == record + } else { + // Not have target_record to check with + true + } + } +} + +impl Debug for GetRecordCfg { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut f = f.debug_struct("GetRecordCfg"); + f.field("get_quorum", &self.get_quorum) + .field("retry_strategy", &self.retry_strategy); + + match &self.target_record { + Some(record) => { + let pretty_key = PrettyPrintRecordKey::from(&record.key); + f.field("target_record", &pretty_key); + } + None => { + f.field("target_record", &"None"); + } + }; + + f.field("expected_holders", &self.expected_holders).finish() + } +} + +/// The various settings related to writing a record to the network. +#[derive(Debug, Clone)] +pub struct PutRecordCfg { + /// The quorum used by KAD PUT. KAD still sends out the request to all the peers set by the `replication_factor`, it + /// just makes sure that we get atleast `n` successful responses defined by the Quorum. + /// Our nodes currently send `Ok()` response for every KAD PUT. Thus this field does not do anything atm. + pub put_quorum: ResponseQuorum, + /// If enabled, the provided `RetryStrategy` is used to retry if a PUT attempt fails. + pub retry_strategy: RetryStrategy, + /// Use the `kad::put_record_to` to PUT the record only to the specified peers. If this option is set to None, we + /// will be using `kad::put_record` which would PUT the record to all the closest members of the record. + pub use_put_record_to: Option>, + /// Enables verification after writing. The VerificationKind is used to determine the method to use. + pub verification: Option<(VerificationKind, GetRecordCfg)>, +} + +/// The methods in which verification on a PUT can be carried out. +#[derive(Debug, Clone)] +pub enum VerificationKind { + /// Uses the default KAD GET to perform verification. + Network, + /// Uses the default KAD GET to perform verification, but don't error out on split records + Crdt, + /// Uses the hash based verification for chunks. + ChunkProof { + expected_proof: ChunkProof, + nonce: Nonce, + }, +} + +#[test] +fn verify_retry_strategy_intervals() { + let intervals = |strategy: RetryStrategy| -> Vec { + let mut backoff = strategy.backoff(); + backoff.set_jitter(0.01); // Make intervals deterministic. + backoff + .into_iter() + .flatten() + .map(|duration| duration.as_secs_f64().round() as u32) + .collect() + }; + + assert_eq!(intervals(RetryStrategy::None), Vec::::new()); + assert_eq!(intervals(RetryStrategy::Quick), vec![2, 4, 8]); + assert_eq!(intervals(RetryStrategy::Balanced), vec![2, 4, 8, 16, 32]); + assert_eq!( + intervals(RetryStrategy::Persistent), + vec![2, 4, 8, 16, 32, 32, 32, 32, 32] + ); + assert_eq!( + intervals(RetryStrategy::N(NonZeroUsize::new(12).unwrap())), + vec![2, 4, 8, 16, 32, 32, 32, 32, 32, 32, 32] + ); +} diff --git a/ant-networking/src/driver.rs b/ant-networking/src/driver.rs index bb1637a099..db91c7e98a 100644 --- a/ant-networking/src/driver.rs +++ b/ant-networking/src/driver.rs @@ -10,6 +10,8 @@ use crate::{ bootstrap::{ContinuousNetworkDiscover, NETWORK_DISCOVER_INTERVAL}, circular_vec::CircularVec, cmd::{LocalSwarmCmd, NetworkSwarmCmd}, + config::GetRecordCfg, + driver::kad::U256, error::{NetworkError, Result}, event::{NetworkEvent, NodeEvent}, external_address::ExternalAddressManager, @@ -21,8 +23,7 @@ use crate::{ record_store_api::UnifiedRecordStore, relay_manager::RelayManager, replication_fetcher::ReplicationFetcher, - target_arch::Interval, - target_arch::{interval, spawn, Instant}, + time::{interval, spawn, Instant, Interval}, transport, GetRecordError, Network, NodeIssue, CLOSE_GROUP_SIZE, }; #[cfg(feature = "open-metrics")] @@ -30,26 +31,21 @@ use crate::{ metrics::service::run_metrics_server, metrics::NetworkMetricsRecorder, MetricsRegistries, }; use ant_bootstrap::BootstrapCacheStore; -use ant_evm::{PaymentQuote, U256}; +use ant_evm::PaymentQuote; use ant_protocol::{ - convert_distance_to_u256, - messages::{ChunkProof, Nonce, Request, Response}, - storage::{try_deserialize_record, RetryStrategy}, + messages::{Request, Response}, version::{ get_network_id, IDENTIFY_CLIENT_VERSION_STR, IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR, REQ_RESPONSE_VERSION_STR, }, - NetworkAddress, PrettyPrintKBucketKey, PrettyPrintRecordKey, + NetworkAddress, PrettyPrintKBucketKey, }; -use ant_registers::SignedRegister; use futures::future::Either; use futures::StreamExt; -#[cfg(feature = "local")] -use libp2p::mdns; -use libp2p::{core::muxing::StreamMuxerBox, relay}; +use libp2p::{core::muxing::StreamMuxerBox, relay, swarm::behaviour::toggle::Toggle}; use libp2p::{ identity::Keypair, - kad::{self, QueryId, Quorum, Record, RecordKey, K_VALUE}, + kad::{self, KBucketDistance as Distance, QueryId, Record, RecordKey, K_VALUE}, multiaddr::Protocol, request_response::{self, Config as RequestResponseConfig, OutboundRequestId, ProtocolSupport}, swarm::{ @@ -72,7 +68,7 @@ use std::{ num::NonZeroUsize, path::PathBuf, }; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{mpsc, oneshot, watch}; use tokio::time::Duration; use tracing::warn; use xor_name::XorName; @@ -140,105 +136,6 @@ const REPLICATION_FACTOR: NonZeroUsize = match NonZeroUsize::new(CLOSE_GROUP_SIZ None => panic!("CLOSE_GROUP_SIZE should not be zero"), }; -/// The various settings to apply to when fetching a record from network -#[derive(Clone)] -pub struct GetRecordCfg { - /// The query will result in an error if we get records less than the provided Quorum - pub get_quorum: Quorum, - /// If enabled, the provided `RetryStrategy` is used to retry if a GET attempt fails. - pub retry_strategy: Option, - /// Only return if we fetch the provided record. - pub target_record: Option, - /// Logs if the record was not fetched from the provided set of peers. - pub expected_holders: HashSet, - /// For register record, only root value shall be checked, not the entire content. - pub is_register: bool, -} - -impl GetRecordCfg { - pub fn does_target_match(&self, record: &Record) -> bool { - if let Some(ref target_record) = self.target_record { - if self.is_register { - let pretty_key = PrettyPrintRecordKey::from(&target_record.key); - - let fetched_register = match try_deserialize_record::(record) { - Ok(fetched_register) => fetched_register, - Err(err) => { - error!("When try to deserialize register from fetched record {pretty_key:?}, have error {err:?}"); - return false; - } - }; - let target_register = match try_deserialize_record::(target_record) - { - Ok(target_register) => target_register, - Err(err) => { - error!("When try to deserialize register from target record {pretty_key:?}, have error {err:?}"); - return false; - } - }; - - target_register.base_register() == fetched_register.base_register() - && target_register.ops() == fetched_register.ops() - } else { - target_record == record - } - } else { - // Not have target_record to check with - true - } - } -} - -impl Debug for GetRecordCfg { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut f = f.debug_struct("GetRecordCfg"); - f.field("get_quorum", &self.get_quorum) - .field("retry_strategy", &self.retry_strategy); - - match &self.target_record { - Some(record) => { - let pretty_key = PrettyPrintRecordKey::from(&record.key); - f.field("target_record", &pretty_key); - } - None => { - f.field("target_record", &"None"); - } - }; - - f.field("expected_holders", &self.expected_holders).finish() - } -} - -/// The various settings related to writing a record to the network. -#[derive(Debug, Clone)] -pub struct PutRecordCfg { - /// The quorum used by KAD PUT. KAD still sends out the request to all the peers set by the `replication_factor`, it - /// just makes sure that we get atleast `n` successful responses defined by the Quorum. - /// Our nodes currently send `Ok()` response for every KAD PUT. Thus this field does not do anything atm. - pub put_quorum: Quorum, - /// If enabled, the provided `RetryStrategy` is used to retry if a PUT attempt fails. - pub retry_strategy: Option, - /// Use the `kad::put_record_to` to PUT the record only to the specified peers. If this option is set to None, we - /// will be using `kad::put_record` which would PUT the record to all the closest members of the record. - pub use_put_record_to: Option>, - /// Enables verification after writing. The VerificationKind is used to determine the method to use. - pub verification: Option<(VerificationKind, GetRecordCfg)>, -} - -/// The methods in which verification on a PUT can be carried out. -#[derive(Debug, Clone)] -pub enum VerificationKind { - /// Uses the default KAD GET to perform verification. - Network, - /// Uses the default KAD GET to perform verification, but don't error out on split records - Crdt, - /// Uses the hash based verification for chunks. - ChunkProof { - expected_proof: ChunkProof, - nonce: Nonce, - }, -} - impl From for NodeEvent { fn from(_: std::convert::Infallible) -> Self { panic!("NodeBehaviour is not Infallible!") @@ -254,10 +151,7 @@ pub(super) struct NodeBehaviour { pub(super) blocklist: libp2p::allow_block_list::Behaviour, pub(super) identify: libp2p::identify::Behaviour, - #[cfg(feature = "local")] - pub(super) mdns: mdns::tokio::Behaviour, - #[cfg(feature = "upnp")] - pub(super) upnp: libp2p::swarm::behaviour::toggle::Toggle, + pub(super) upnp: Toggle, pub(super) relay_client: libp2p::relay::client::Behaviour, pub(super) relay_server: libp2p::relay::Behaviour, pub(super) kademlia: kad::Behaviour, @@ -277,7 +171,6 @@ pub struct NetworkBuilder { #[cfg(feature = "open-metrics")] metrics_server_port: Option, request_timeout: Option, - #[cfg(feature = "upnp")] upnp: bool, } @@ -295,7 +188,6 @@ impl NetworkBuilder { #[cfg(feature = "open-metrics")] metrics_server_port: None, request_timeout: None, - #[cfg(feature = "upnp")] upnp: false, } } @@ -333,7 +225,6 @@ impl NetworkBuilder { self.metrics_server_port = port; } - #[cfg(feature = "upnp")] pub fn upnp(&mut self, upnp: bool) { self.upnp = upnp; } @@ -423,17 +314,10 @@ impl NetworkBuilder { }; let listen_addr = self.listen_addr; - #[cfg(feature = "upnp")] let upnp = self.upnp; - let (network, events_receiver, mut swarm_driver) = self.build( - kad_cfg, - Some(store_cfg), - false, - ProtocolSupport::Full, - #[cfg(feature = "upnp")] - upnp, - )?; + let (network, events_receiver, mut swarm_driver) = + self.build(kad_cfg, Some(store_cfg), false, ProtocolSupport::Full, upnp); // Listen on the provided address let listen_socket_addr = listen_addr.ok_or(NetworkError::ListenAddressNotProvided)?; @@ -450,7 +334,7 @@ impl NetworkBuilder { } /// Same as `build_node` API but creates the network components in client mode - pub fn build_client(self) -> Result<(Network, mpsc::Receiver, SwarmDriver)> { + pub fn build_client(self) -> (Network, mpsc::Receiver, SwarmDriver) { // Create a Kademlia behaviour for client mode, i.e. set req/resp protocol // to outbound-only mode and don't listen on any address let mut kad_cfg = kad::Config::new(KAD_STREAM_PROTOCOL_ID); // default query timeout is 60 secs @@ -465,16 +349,10 @@ impl NetworkBuilder { // How many nodes _should_ store data. .set_replication_factor(REPLICATION_FACTOR); - let (network, net_event_recv, driver) = self.build( - kad_cfg, - None, - true, - ProtocolSupport::Outbound, - #[cfg(feature = "upnp")] - false, - )?; + let (network, net_event_recv, driver) = + self.build(kad_cfg, None, true, ProtocolSupport::Outbound, false); - Ok((network, net_event_recv, driver)) + (network, net_event_recv, driver) } /// Private helper to create the network components with the provided config and req/res behaviour @@ -484,8 +362,8 @@ impl NetworkBuilder { record_store_cfg: Option, is_client: bool, req_res_protocol: ProtocolSupport, - #[cfg(feature = "upnp")] upnp: bool, - ) -> Result<(Network, mpsc::Receiver, SwarmDriver)> { + upnp: bool, + ) -> (Network, mpsc::Receiver, SwarmDriver) { let identify_protocol_str = IDENTIFY_PROTOCOL_STR .read() .expect("Failed to obtain read lock for IDENTIFY_PROTOCOL_STR") @@ -493,7 +371,6 @@ impl NetworkBuilder { let peer_id = PeerId::from(self.keypair.public()); // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): - #[cfg(not(target_arch = "wasm32"))] info!( "Process (PID: {}) with PeerId: {peer_id}", std::process::id() @@ -594,19 +471,17 @@ impl NetworkBuilder { let kademlia = { match record_store_cfg { Some(store_cfg) => { + #[cfg(feature = "open-metrics")] + let record_stored_metrics = + metrics_recorder.as_ref().map(|r| r.records_stored.clone()); let node_record_store = NodeRecordStore::with_config( peer_id, store_cfg, network_event_sender.clone(), local_swarm_cmd_sender.clone(), + #[cfg(feature = "open-metrics")] + record_stored_metrics, ); - #[cfg(feature = "open-metrics")] - let mut node_record_store = node_record_store; - #[cfg(feature = "open-metrics")] - if let Some(metrics_recorder) = &metrics_recorder { - node_record_store = node_record_store - .set_record_count_metric(metrics_recorder.records_stored.clone()); - } let store = UnifiedRecordStore::Node(node_record_store); debug!("Using Kademlia with NodeRecordStore!"); @@ -621,18 +496,6 @@ impl NetworkBuilder { } }; - #[cfg(feature = "local")] - let mdns_config = mdns::Config { - // lower query interval to speed up peer discovery - // this increases traffic, but means we no longer have clients unable to connect - // after a few minutes - query_interval: Duration::from_secs(5), - ..Default::default() - }; - - #[cfg(feature = "local")] - let mdns = mdns::tokio::Behaviour::new(mdns_config, peer_id)?; - let agent_version = if is_client { IDENTIFY_CLIENT_VERSION_STR .read() @@ -650,11 +513,11 @@ impl NetworkBuilder { let cfg = libp2p::identify::Config::new(identify_protocol_str, self.keypair.public()) .with_agent_version(agent_version) // Enlength the identify interval from default 5 mins to 1 hour. - .with_interval(RESEND_IDENTIFY_INVERVAL); + .with_interval(RESEND_IDENTIFY_INVERVAL) + .with_hide_listen_addrs(true); libp2p::identify::Behaviour::new(cfg) }; - #[cfg(feature = "upnp")] let upnp = if !self.local && !is_client && upnp { debug!("Enabling UPnP port opening behavior"); Some(libp2p::upnp::tokio::Behaviour::default()) @@ -680,21 +543,14 @@ impl NetworkBuilder { blocklist: libp2p::allow_block_list::Behaviour::default(), relay_client: relay_behaviour, relay_server, - #[cfg(feature = "upnp")] upnp, request_response, kademlia, identify, - #[cfg(feature = "local")] - mdns, }; - #[cfg(not(target_arch = "wasm32"))] let swarm_config = libp2p::swarm::Config::with_tokio_executor() .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); - #[cfg(target_arch = "wasm32")] - let swarm_config = libp2p::swarm::Config::with_wasm_executor() - .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); let swarm = Swarm::new(transport, behaviour, peer_id, swarm_config); @@ -704,6 +560,14 @@ impl NetworkBuilder { // Enable relay manager for nodes behind home network let relay_manager = if !is_client && self.is_behind_home_network { let relay_manager = RelayManager::new(peer_id); + #[cfg(feature = "open-metrics")] + let mut relay_manager = relay_manager; + #[cfg(feature = "open-metrics")] + if let Some(metrics_recorder) = &metrics_recorder { + relay_manager.set_reservation_health_metrics( + metrics_recorder.relay_reservation_health.clone(), + ); + } Some(relay_manager) } else { info!("Relay manager is disabled for this node."); @@ -770,7 +634,7 @@ impl NetworkBuilder { self.keypair, ); - Ok((network, network_event_receiver, swarm_driver)) + (network, network_event_receiver, swarm_driver) } } @@ -881,7 +745,7 @@ impl SwarmDriver { /// The `tokio::select` macro is used to concurrently process swarm events /// and command receiver messages, ensuring efficient handling of multiple /// asynchronous tasks. - pub async fn run(mut self) { + pub async fn run(mut self, mut shutdown_rx: watch::Receiver) { let mut network_discover_interval = interval(NETWORK_DISCOVER_INTERVAL); let mut set_farthest_record_interval = interval(CLOSET_RECORD_CHECK_INTERVAL); let mut relay_manager_reservation_interval = interval(RELAY_MANAGER_RESERVATION_INTERVAL); @@ -936,6 +800,13 @@ impl SwarmDriver { }, None => continue, }, + // Check for a shutdown command. + result = shutdown_rx.changed() => { + if result.is_ok() && *shutdown_rx.borrow() || result.is_err() { + info!("Shutdown signal received or sender dropped. Exiting swarm driver loop."); + break; + } + }, // next take and react to external swarm events swarm_event = self.swarm.select_next_some() => { // Refer to the handle_swarm_events::IncomingConnectionError for more info on why we skip @@ -1000,9 +871,8 @@ impl SwarmDriver { // Note: self is included let self_addr = NetworkAddress::from_peer(self.self_peer_id); let close_peers_distance = self_addr.distance(&NetworkAddress::from_peer(closest_k_peers[CLOSE_GROUP_SIZE + 1])); - let close_peers_u256 = convert_distance_to_u256(&close_peers_distance); - let distance = std::cmp::max(density_distance, close_peers_u256); + let distance = std::cmp::max(Distance(density_distance), close_peers_distance); // The sampling approach has severe impact to the node side performance // Hence suggested to be only used by client side. @@ -1021,7 +891,7 @@ impl SwarmDriver { // self_addr.distance(&NetworkAddress::from_peer(closest_k_peers[CLOSE_GROUP_SIZE])) // }; - info!("Set responsible range to {distance:?}({:?})", distance.log2()); + info!("Set responsible range to {distance:?}({:?})", distance.ilog2()); // set any new distance to farthest record in the store self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); @@ -1075,9 +945,7 @@ impl SwarmDriver { let new_duration = Duration::from_secs(std::cmp::min(scaled, max_cache_save_duration.as_secs())); info!("Scaling up the bootstrap cache save interval to {new_duration:?}"); - // `Interval` ticks immediately for Tokio, but not for `wasmtimer`, which is used for wasm32. *current_interval = interval(new_duration); - #[cfg(not(target_arch = "wasm32"))] current_interval.tick().await; trace!("Bootstrap cache synced in {:?}", start.elapsed()); @@ -1134,23 +1002,22 @@ impl SwarmDriver { /// get closest k_value the peers from our local RoutingTable. Contains self. /// Is sorted for closeness to self. pub(crate) fn get_closest_k_value_local_peers(&mut self) -> Vec { - let self_peer_id = self.self_peer_id.into(); + let k_bucket_key = NetworkAddress::from_peer(self.self_peer_id).as_kbucket_key(); // get closest peers from buckets, sorted by increasing distance to us - let peers = self + let peers: Vec<_> = self .swarm .behaviour_mut() .kademlia - .get_closest_local_peers(&self_peer_id) + .get_closest_local_peers(&k_bucket_key) // Map KBucketKey to PeerId. - .map(|key| key.into_preimage()); + .map(|key| key.into_preimage()) + // Limit ourselves to K_VALUE (20) peers. + .take(K_VALUE.get() - 1) + .collect(); // Start with our own PeerID and chain the closest. - std::iter::once(self.self_peer_id) - .chain(peers) - // Limit ourselves to K_VALUE (20) peers. - .take(K_VALUE.get()) - .collect() + std::iter::once(self.self_peer_id).chain(peers).collect() } /// Dials the given multiaddress. If address contains a peer ID, simultaneous diff --git a/ant-networking/src/error.rs b/ant-networking/src/error.rs index c683ff4432..c9444d1f59 100644 --- a/ant-networking/src/error.rs +++ b/ant-networking/src/error.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_protocol::storage::TransactionAddress; +use ant_protocol::storage::GraphEntryAddress; use ant_protocol::{messages::Response, storage::RecordKind, NetworkAddress, PrettyPrintRecordKey}; use libp2p::{ kad::{self, QueryId, Record}, @@ -45,8 +45,7 @@ pub enum GetRecordError { RecordNotFound, // Avoid logging the whole `Record` content by accident. /// The split record error will be handled at the network layer. - /// For transactions, it accumulates the transactions - /// For registers, it merges the registers and returns the merged record. + /// For GraphEntry, it accumulates them #[error("Split Record has {} different copies", result_map.len())] SplitRecord { result_map: HashMap)>, @@ -123,21 +122,21 @@ pub enum NetworkError { #[error("Record header is incorrect")] InCorrectRecordHeader, - // ---------- Transfer Errors - #[error("Failed to get transaction: {0}")] - FailedToGetSpend(String), - #[error("Transfer is invalid: {0}")] - InvalidTransfer(String), + #[error("The operation is not allowed on a client record store")] + OperationNotAllowedOnClientRecordStore, // ---------- Chunk Errors #[error("Failed to verify the ChunkProof with the provided quorum")] FailedToVerifyChunkProof(NetworkAddress), - // ---------- Transaction Errors - #[error("Transaction not found: {0:?}")] - NoTransactionFoundInsideRecord(TransactionAddress), + // ---------- Graph Errors + #[error("Graph entry not found: {0:?}")] + NoGraphEntryFoundInsideRecord(GraphEntryAddress), // ---------- Store Error + #[error("Not Enough Peers for Store Cost Request")] + NotEnoughPeersForStoreCostRequest, + #[error("No Store Cost Responses")] NoStoreCostResponses, @@ -179,9 +178,6 @@ pub enum NetworkError { #[error("Error setting up behaviour: {0}")] BehaviourErr(String), - - #[error("Register already exists at this address")] - RegisterAlreadyExists, } #[cfg(test)] diff --git a/ant-networking/src/event/identify.rs b/ant-networking/src/event/identify.rs new file mode 100644 index 0000000000..c06eac14e5 --- /dev/null +++ b/ant-networking/src/event/identify.rs @@ -0,0 +1,225 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::relay_manager::is_a_relayed_peer; +use crate::{multiaddr_is_global, multiaddr_strip_p2p, NetworkEvent, SwarmDriver}; +use ant_protocol::version::{IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR}; +use libp2p::identify::Info; +use libp2p::kad::K_VALUE; +use libp2p::multiaddr::Protocol; +use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; +use libp2p::Multiaddr; +use std::collections::HashSet; +use std::time::Instant; + +impl SwarmDriver { + pub(super) fn handle_identify_event(&mut self, identify_event: libp2p::identify::Event) { + match identify_event { + libp2p::identify::Event::Received { + peer_id, + info, + connection_id, + } => { + let start = Instant::now(); + self.handle_identify_received(peer_id, info, connection_id); + trace!("SwarmEvent handled in {:?}: identify", start.elapsed()); + } + // Log the other Identify events. + libp2p::identify::Event::Sent { .. } => debug!("identify: {identify_event:?}"), + libp2p::identify::Event::Pushed { .. } => debug!("identify: {identify_event:?}"), + libp2p::identify::Event::Error { .. } => warn!("identify: {identify_event:?}"), + } + } + + fn handle_identify_received( + &mut self, + peer_id: libp2p::PeerId, + info: Info, + connection_id: libp2p::swarm::ConnectionId, + ) { + debug!(conn_id=%connection_id, %peer_id, ?info, "identify: received info"); + + let our_identify_protocol = IDENTIFY_PROTOCOL_STR.read().expect("IDENTIFY_PROTOCOL_STR has been locked to write. A call to set_network_id performed. This should not happen.").to_string(); + + if info.protocol_version != our_identify_protocol { + warn!(?info.protocol_version, "identify: {peer_id:?} does not have the same protocol. Our IDENTIFY_PROTOCOL_STR: {our_identify_protocol:?}"); + + self.send_event(NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol: our_identify_protocol, + their_protocol: info.protocol_version, + }); + // Block the peer from any further communication. + self.swarm.behaviour_mut().blocklist.block_peer(peer_id); + if let Some(dead_peer) = self.swarm.behaviour_mut().kademlia.remove_peer(&peer_id) { + error!("Clearing out a protocol mismatch peer from RT. The peer pushed an incorrect identify info after being added: {peer_id:?}"); + self.update_on_peer_removal(*dead_peer.node.key.preimage()); + } + + return; + } + + let our_agent_version = IDENTIFY_NODE_VERSION_STR.read().expect("IDENTIFY_NODE_VERSION_STR has been locked to write. A call to set_network_id performed. This should not happen.").to_string(); + // if client, return. + if info.agent_version != our_agent_version { + return; + } + + let has_dialed = self.dialed_peers.contains(&peer_id); + + // If we're not in local mode, only add globally reachable addresses. + // Strip the `/p2p/...` part of the multiaddresses. + // Collect into a HashSet directly to avoid multiple allocations and handle deduplication. + let mut addrs: HashSet = match self.local { + true => info + .listen_addrs + .into_iter() + .map(|addr| multiaddr_strip_p2p(&addr)) + .collect(), + false => info + .listen_addrs + .into_iter() + .filter(multiaddr_is_global) + .map(|addr| multiaddr_strip_p2p(&addr)) + .collect(), + }; + + let is_relayed_peer = is_a_relayed_peer(&addrs); + + let is_bootstrap_peer = self + .bootstrap_peers + .iter() + .any(|(_ilog2, peers)| peers.contains(&peer_id)); + + // Do not use an `already relayed` peer as `potential relay candidate`. + if !is_relayed_peer && !is_bootstrap_peer { + if let Some(relay_manager) = self.relay_manager.as_mut() { + debug!("Adding candidate relay server {peer_id:?}, it's not a bootstrap node"); + relay_manager.add_potential_candidates(&peer_id, &addrs, &info.protocols); + } + } + + let (kbucket_full, already_present_in_rt, ilog2) = + if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { + let ilog2 = kbucket.range().0.ilog2(); + let num_peers = kbucket.num_entries(); + let is_bucket_full = num_peers >= K_VALUE.into(); + + // check if peer_id is already a part of RT + let already_present_in_rt = kbucket + .iter() + .any(|entry| entry.node.key.preimage() == &peer_id); + + (is_bucket_full, already_present_in_rt, ilog2) + } else { + return; + }; + + // If the peer is part already of the RT, try updating the addresses based on the new push info. + // We don't have to dial it back. + if already_present_in_rt { + debug!("Received identify for {peer_id:?} that is already part of the RT. Checking if the addresses {addrs:?} are new."); + + self.update_pre_existing_peer(peer_id, addrs.clone()); + return; + } + + // When received an identify from un-dialed peer, try to dial it + // The dial shall trigger the same identify to be sent again and confirm + // peer is external accessible, hence safe to be added into RT. + if !self.local && !has_dialed { + // Only need to dial back for not fulfilled kbucket + if kbucket_full { + debug!("received identify for a full bucket {ilog2:?}, not dialing {peer_id:?} on {addrs:?}"); + return; + } + + info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); + if let Err(err) = self.swarm.dial( + DialOpts::peer_id(peer_id) + .condition(PeerCondition::NotDialing) + .addresses(addrs.iter().cloned().collect()) + .build(), + ) { + warn!(%peer_id, ?addrs, "dialing error: {err:?}"); + } + } else + // We care only for peers that we dialed and thus are reachable. + // Or if we are local, we can add the peer directly. + { + // A bad node cannot establish a connection with us. So we can add it to the RT directly. + + // With the new bootstrap cache, the workload is distributed, + // hence no longer need to replace bootstrap nodes for workload share. + // self.remove_bootstrap_from_full(peer_id); + + // Avoid have `direct link format` addrs co-exists with `relay` addr + if is_relayed_peer { + addrs.retain(|multiaddr| { + multiaddr.iter().any(|p| matches!(p, Protocol::P2pCircuit)) + }); + } + + debug!(%peer_id, ?addrs, "identify: attempting to add addresses to routing table"); + + // Attempt to add the addresses to the routing table. + for multiaddr in addrs { + let _routing_update = self + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer_id, multiaddr); + } + } + } + + /// If the peer is part already of the RT, try updating the addresses based on the new push info. + fn update_pre_existing_peer(&mut self, peer_id: libp2p::PeerId, new_addrs: HashSet) { + if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { + let mut addresses_to_add = Vec::new(); + let mut addresses_to_remove = Vec::new(); + + let Some(entry) = kbucket + .iter() + .find(|entry| entry.node.key.preimage() == &peer_id) + else { + warn!("Peer {peer_id:?} is not part of the RT. Cannot update addresses."); + return; + }; + let existing_addrs = entry + .node + .value + .iter() + .map(multiaddr_strip_p2p) + .collect::>(); + addresses_to_add.extend(new_addrs.difference(&existing_addrs)); + addresses_to_remove.extend(existing_addrs.difference(&new_addrs)); + + if !addresses_to_remove.is_empty() { + debug!("Removing addresses from RT for {peer_id:?} as the new identify does not contain them: {addresses_to_remove:?}"); + for multiaddr in addresses_to_remove { + let _routing_update = self + .swarm + .behaviour_mut() + .kademlia + .remove_address(&peer_id, multiaddr); + } + } + + if !addresses_to_add.is_empty() { + debug!("Adding addresses to RT for {peer_id:?} as the new identify contains them: {addresses_to_add:?}"); + for multiaddr in addresses_to_add { + let _routing_update = self + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer_id, multiaddr.clone()); + } + } + } + } +} diff --git a/ant-networking/src/event/kad.rs b/ant-networking/src/event/kad.rs index 1af95f9d1d..6d95016942 100644 --- a/ant-networking/src/event/kad.rs +++ b/ant-networking/src/event/kad.rs @@ -7,12 +7,11 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - driver::PendingGetClosestType, get_quorum_value, get_transactions_from_record, - target_arch::Instant, GetRecordCfg, GetRecordError, NetworkError, Result, SwarmDriver, - CLOSE_GROUP_SIZE, + driver::PendingGetClosestType, get_graph_entry_from_record, time::Instant, GetRecordCfg, + GetRecordError, NetworkError, Result, SwarmDriver, CLOSE_GROUP_SIZE, }; use ant_protocol::{ - storage::{try_serialize_record, RecordKind, Transaction}, + storage::{try_serialize_record, DataTypes, GraphEntry, RecordKind}, NetworkAddress, PrettyPrintRecordKey, }; use itertools::Itertools; @@ -381,7 +380,7 @@ impl SwarmDriver { 1 }; - let expected_answers = get_quorum_value(&cfg.get_quorum); + let expected_answers = cfg.get_quorum.get_value(); debug!("Expecting {expected_answers:?} answers for record {pretty_key:?} task {query_id:?}, received {responded_peers} so far"); if responded_peers >= expected_answers { @@ -397,26 +396,26 @@ impl SwarmDriver { Self::send_record_after_checking_target(senders, peer_record.record, &cfg)?; } else { debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with split record"); - let mut accumulated_transactions = BTreeSet::new(); + let mut accumulated_graph_entries = BTreeSet::new(); for (record, _) in result_map.values() { - match get_transactions_from_record(record) { - Ok(transactions) => { - accumulated_transactions.extend(transactions); + match get_graph_entry_from_record(record) { + Ok(graph_entries) => { + accumulated_graph_entries.extend(graph_entries); } Err(_) => { continue; } } } - if !accumulated_transactions.is_empty() { - info!("For record {pretty_key:?} task {query_id:?}, found split record for a transaction, accumulated and sending them as a single record"); - let accumulated_transactions = accumulated_transactions + if !accumulated_graph_entries.is_empty() { + info!("For record {pretty_key:?} task {query_id:?}, found split record for a GraphEntry, accumulated and sending them as a single record"); + let accumulated_graph_entries = accumulated_graph_entries .into_iter() - .collect::>(); + .collect::>(); let bytes = try_serialize_record( - &accumulated_transactions, - RecordKind::Transaction, + &accumulated_graph_entries, + RecordKind::DataOnly(DataTypes::GraphEntry), )?; let new_accumulated_record = Record { @@ -507,12 +506,12 @@ impl SwarmDriver { let result = if let Some((record, peers)) = result_map.values().next() { trace!("one version found for record {data_key_address:?}!"); - if peers.len() >= get_quorum_value(&cfg.get_quorum) { + if peers.len() >= cfg.get_quorum.get_value() { Ok(record.clone()) } else { Err(GetRecordError::NotEnoughCopies { record: record.clone(), - expected: get_quorum_value(&cfg.get_quorum), + expected: cfg.get_quorum.get_value(), got: peers.len(), }) } @@ -583,7 +582,7 @@ impl SwarmDriver { } })?; - let required_response_count = get_quorum_value(&cfg.get_quorum); + let required_response_count = cfg.get_quorum.get_value(); // if we've a split over the result xorname, then we don't attempt to resolve this here. // Retry and resolve through normal flows without a timeout. diff --git a/ant-networking/src/event/mod.rs b/ant-networking/src/event/mod.rs index ae6e2aefca..2ad32c079f 100644 --- a/ant-networking/src/event/mod.rs +++ b/ant-networking/src/event/mod.rs @@ -6,6 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +mod identify; mod kad; mod request_response; mod swarm; @@ -13,25 +14,25 @@ mod swarm; use crate::{driver::SwarmDriver, error::Result}; use core::fmt; use custom_debug::Debug as CustomDebug; -#[cfg(feature = "local")] -use libp2p::mdns; use libp2p::{ kad::{Addresses, Record, RecordKey, K_VALUE}, request_response::ResponseChannel as PeerResponseChannel, Multiaddr, PeerId, }; -use ant_evm::PaymentQuote; +use ant_evm::{PaymentQuote, ProofOfPayment}; +use ant_protocol::storage::DataTypes; #[cfg(feature = "open-metrics")] use ant_protocol::CLOSE_GROUP_SIZE; use ant_protocol::{ messages::{Query, Request, Response}, + storage::ValidationType, NetworkAddress, PrettyPrintRecordKey, }; #[cfg(feature = "open-metrics")] use std::collections::HashSet; use std::{ - collections::BTreeSet, + collections::BTreeMap, fmt::{Debug, Formatter}, }; use tokio::sync::oneshot; @@ -42,19 +43,15 @@ type KBucketStatus = (usize, usize, usize, usize, Vec<(usize, usize, u32)>); /// NodeEvent enum #[derive(CustomDebug)] pub(super) enum NodeEvent { - #[cfg(feature = "upnp")] Upnp(libp2p::upnp::Event), MsgReceived(libp2p::request_response::Event), Kademlia(libp2p::kad::Event), - #[cfg(feature = "local")] - Mdns(Box), Identify(Box), RelayClient(Box), RelayServer(Box), Void(void::Void), } -#[cfg(feature = "upnp")] impl From for NodeEvent { fn from(event: libp2p::upnp::Event) -> Self { NodeEvent::Upnp(event) @@ -73,13 +70,6 @@ impl From for NodeEvent { } } -#[cfg(feature = "local")] -impl From for NodeEvent { - fn from(event: mdns::Event) -> Self { - NodeEvent::Mdns(Box::new(event)) - } -} - impl From for NodeEvent { fn from(event: libp2p::identify::Event) -> Self { NodeEvent::Identify(Box::new(event)) @@ -143,9 +133,19 @@ pub enum NetworkEvent { /// Terminate Node on unrecoverable errors TerminateNode { reason: TerminateNodeReason }, /// List of peer nodes that failed to fetch replication copy from. - FailedToFetchHolders(BTreeSet), + FailedToFetchHolders(BTreeMap), /// Quotes to be verified QuoteVerification { quotes: Vec<(PeerId, PaymentQuote)> }, + /// Fresh replicate to fetch + FreshReplicateToFetch { + holder: NetworkAddress, + keys: Vec<( + NetworkAddress, + DataTypes, + ValidationType, + Option, + )>, + }, } /// Terminate node for the following reason @@ -204,6 +204,12 @@ impl Debug for NetworkEvent { quotes.len() ) } + NetworkEvent::FreshReplicateToFetch { holder, keys } => { + write!( + f, + "NetworkEvent::FreshReplicateToFetch({holder:?}, {keys:?})" + ) + } } } } diff --git a/ant-networking/src/event/request_response.rs b/ant-networking/src/event/request_response.rs index ce6755e8dc..d871e48488 100644 --- a/ant-networking/src/event/request_response.rs +++ b/ant-networking/src/event/request_response.rs @@ -12,7 +12,7 @@ use crate::{ }; use ant_protocol::{ messages::{CmdResponse, Request, Response}, - storage::RecordType, + storage::ValidationType, NetworkAddress, }; use libp2p::request_response::{self, Message}; @@ -24,7 +24,7 @@ impl SwarmDriver { event: request_response::Event, ) -> Result<(), NetworkError> { match event { - request_response::Event::Message { message, peer } => match message { + request_response::Event::Message { message, peer, .. } => match message { Message::Request { request, channel, @@ -46,7 +46,22 @@ impl SwarmDriver { channel: MsgResponder::FromPeer(channel), }); - self.add_keys_to_replication_fetcher(holder, keys); + self.add_keys_to_replication_fetcher(holder, keys, false)?; + } + Request::Cmd(ant_protocol::messages::Cmd::FreshReplicate { + holder, + keys, + }) => { + let response = Response::Cmd( + ant_protocol::messages::CmdResponse::FreshReplicate(Ok(())), + ); + + self.queue_network_swarm_cmd(NetworkSwarmCmd::SendResponse { + resp: response, + channel: MsgResponder::FromPeer(channel), + }); + + self.send_event(NetworkEvent::FreshReplicateToFetch { holder, keys }); } Request::Cmd(ant_protocol::messages::Cmd::PeerConsideredAsBad { detected_by, @@ -124,6 +139,7 @@ impl SwarmDriver { request_id, error, peer, + .. } => { if let Some(sender) = self.pending_requests.remove(&request_id) { match sender { @@ -146,30 +162,34 @@ impl SwarmDriver { peer, request_id, error, + .. } => { warn!("RequestResponse: InboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); } - request_response::Event::ResponseSent { peer, request_id } => { + request_response::Event::ResponseSent { + peer, request_id, .. + } => { debug!("ResponseSent for request_id: {request_id:?} and peer: {peer:?}"); } } Ok(()) } - fn add_keys_to_replication_fetcher( + pub(crate) fn add_keys_to_replication_fetcher( &mut self, sender: NetworkAddress, - incoming_keys: Vec<(NetworkAddress, RecordType)>, - ) { + incoming_keys: Vec<(NetworkAddress, ValidationType)>, + is_fresh_replicate: bool, + ) -> Result<(), NetworkError> { let holder = if let Some(peer_id) = sender.as_peer_id() { peer_id } else { warn!("Replication list sender is not a peer_id {sender:?}"); - return; + return Ok(()); }; debug!( - "Received replication list from {holder:?} of {} keys", + "Received replication list from {holder:?} of {} keys is_fresh_replicate {is_fresh_replicate:?}", incoming_keys.len() ); @@ -178,27 +198,36 @@ impl SwarmDriver { let closest_k_peers = self.get_closest_k_value_local_peers(); if !closest_k_peers.contains(&holder) || holder == self.self_peer_id { debug!("Holder {holder:?} is self or not in replication range."); - return; + return Ok(()); } // On receive a replication_list from a close_group peer, we undertake: // 1, For those keys that we don't have: // fetch them if close enough to us - // 2, For those transactions that we have that differ in the hash, we fetch the other version + // 2, For those GraphEntry that we have that differ in the hash, we fetch the other version // and update our local copy. let all_keys = self .swarm .behaviour_mut() .kademlia .store_mut() - .record_addresses_ref(); - let keys_to_fetch = self - .replication_fetcher - .add_keys(holder, incoming_keys, all_keys); + .record_addresses_ref()?; + let keys_to_fetch = self.replication_fetcher.add_keys( + holder, + incoming_keys, + all_keys, + is_fresh_replicate, + closest_k_peers + .iter() + .map(|peer_id| NetworkAddress::from_peer(*peer_id)) + .collect(), + ); if keys_to_fetch.is_empty() { debug!("no waiting keys to fetch from the network"); } else { self.send_event(NetworkEvent::KeysToFetchForReplication(keys_to_fetch)); } + + Ok(()) } } diff --git a/ant-networking/src/event/swarm.rs b/ant-networking/src/event/swarm.rs index b37165bd50..599a97ab91 100644 --- a/ant-networking/src/event/swarm.rs +++ b/ant-networking/src/event/swarm.rs @@ -7,25 +7,19 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - event::NodeEvent, multiaddr_get_ip, multiaddr_is_global, multiaddr_strip_p2p, - relay_manager::is_a_relayed_peer, target_arch::Instant, NetworkEvent, Result, SwarmDriver, + event::NodeEvent, multiaddr_get_ip, time::Instant, NetworkEvent, NodeIssue, Result, SwarmDriver, }; -use ant_protocol::version::{IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR}; -#[cfg(feature = "local")] -use libp2p::mdns; +use ant_bootstrap::BootstrapCacheStore; +use itertools::Itertools; #[cfg(feature = "open-metrics")] use libp2p::metrics::Recorder; use libp2p::{ core::ConnectedPoint, kad::K_VALUE, multiaddr::Protocol, - swarm::{ - dial_opts::{DialOpts, PeerCondition}, - ConnectionId, DialError, SwarmEvent, - }, + swarm::{ConnectionId, DialError, SwarmEvent}, Multiaddr, PeerId, TransportError, }; -use std::collections::HashSet; use tokio::time::Duration; impl SwarmDriver { @@ -55,21 +49,33 @@ impl SwarmDriver { self.handle_kad_event(kad_event)?; } SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { + #[cfg(feature = "open-metrics")] + if let Some(metrics_recorder) = &self.metrics_recorder { + metrics_recorder.record(&(*event)); + } event_string = "relay_client_event"; info!(?event, "relay client event"); if let libp2p::relay::client::Event::ReservationReqAccepted { - relay_peer_id, .. + relay_peer_id, + renewal, + .. } = *event { - if let Some(relay_manager) = self.relay_manager.as_mut() { - relay_manager - .on_successful_reservation_by_client(&relay_peer_id, &mut self.swarm); + if !renewal { + if let Some(relay_manager) = self.relay_manager.as_mut() { + relay_manager.on_successful_reservation_by_client( + &relay_peer_id, + &mut self.swarm, + &self.live_connected_peers, + ); + } + } else { + info!("Relay reservation was renewed with {relay_peer_id:?}"); } } } - #[cfg(feature = "upnp")] SwarmEvent::Behaviour(NodeEvent::Upnp(upnp_event)) => { #[cfg(feature = "open-metrics")] if let Some(metrics_recorder) = &self.metrics_recorder { @@ -108,207 +114,14 @@ impl SwarmDriver { _ => {} } } - SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { + SwarmEvent::Behaviour(NodeEvent::Identify(event)) => { // Record the Identify event for metrics if the feature is enabled. #[cfg(feature = "open-metrics")] if let Some(metrics_recorder) = &self.metrics_recorder { - metrics_recorder.record(&(*iden)); + metrics_recorder.record(&(*event)); } event_string = "identify"; - - match *iden { - libp2p::identify::Event::Received { - peer_id, - info, - connection_id, - } => { - debug!(conn_id=%connection_id, %peer_id, ?info, "identify: received info"); - - let our_identify_protocol = IDENTIFY_PROTOCOL_STR.read().expect("IDENTIFY_PROTOCOL_STR has been locked to write. A call to set_network_id performed. This should not happen.").to_string(); - - if info.protocol_version != our_identify_protocol { - warn!(?info.protocol_version, "identify: {peer_id:?} does not have the same protocol. Our IDENTIFY_PROTOCOL_STR: {our_identify_protocol:?}"); - - self.send_event(NetworkEvent::PeerWithUnsupportedProtocol { - our_protocol: our_identify_protocol, - their_protocol: info.protocol_version, - }); - // Block the peer from any further communication. - self.swarm.behaviour_mut().blocklist.block_peer(peer_id); - if let Some(dead_peer) = - self.swarm.behaviour_mut().kademlia.remove_peer(&peer_id) - { - error!("Clearing out a protocol mistmatch peer from RT. Something went wrong, we should not have added this peer to RT: {peer_id:?}"); - self.update_on_peer_removal(*dead_peer.node.key.preimage()); - } - - return Ok(()); - } - - let our_agent_version = IDENTIFY_NODE_VERSION_STR.read().expect("IDENTIFY_NODE_VERSION_STR has been locked to write. A call to set_network_id performed. This should not happen.").to_string(); - // if client, return. - if info.agent_version != our_agent_version { - return Ok(()); - } - - let has_dialed = self.dialed_peers.contains(&peer_id); - - // If we're not in local mode, only add globally reachable addresses. - // Strip the `/p2p/...` part of the multiaddresses. - // Collect into a HashSet directly to avoid multiple allocations and handle deduplication. - let mut addrs: HashSet = match self.local { - true => info - .listen_addrs - .into_iter() - .map(|addr| multiaddr_strip_p2p(&addr)) - .collect(), - false => info - .listen_addrs - .into_iter() - .filter(multiaddr_is_global) - .map(|addr| multiaddr_strip_p2p(&addr)) - .collect(), - }; - - let has_relayed = is_a_relayed_peer(&addrs); - - let is_bootstrap_peer = self - .bootstrap_peers - .iter() - .any(|(_ilog2, peers)| peers.contains(&peer_id)); - - // Do not use an `already relayed` peer as `potential relay candidate`. - if !has_relayed && !is_bootstrap_peer { - if let Some(relay_manager) = self.relay_manager.as_mut() { - debug!("Adding candidate relay server {peer_id:?}, it's not a bootstrap node"); - relay_manager.add_potential_candidates( - &peer_id, - &addrs, - &info.protocols, - ); - } - } - - // When received an identify from un-dialed peer, try to dial it - // The dial shall trigger the same identify to be sent again and confirm - // peer is external accessible, hence safe to be added into RT. - if !self.local && !has_dialed { - // Only need to dial back for not fulfilled kbucket - let (kbucket_full, already_present_in_rt, ilog2) = - if let Some(kbucket) = - self.swarm.behaviour_mut().kademlia.kbucket(peer_id) - { - let ilog2 = kbucket.range().0.ilog2(); - let num_peers = kbucket.num_entries(); - let is_bucket_full = num_peers >= K_VALUE.into(); - - // check if peer_id is already a part of RT - let already_present_in_rt = kbucket - .iter() - .any(|entry| entry.node.key.preimage() == &peer_id); - - // // If the bucket contains any of a bootstrap node, - // // consider the bucket is not full and dial back - // // so that the bootstrap nodes can be replaced. - // if is_bucket_full { - // if let Some(peers) = self.bootstrap_peers.get(&ilog2) { - // if kbucket.iter().any(|entry| { - // peers.contains(entry.node.key.preimage()) - // }) { - // is_bucket_full = false; - // } - // } - // } - - (is_bucket_full, already_present_in_rt, ilog2) - } else { - return Ok(()); - }; - - if kbucket_full { - debug!("received identify for a full bucket {ilog2:?}, not dialing {peer_id:?} on {addrs:?}"); - return Ok(()); - } else if already_present_in_rt { - debug!("received identify for {peer_id:?} that is already part of the RT. Not dialing {peer_id:?} on {addrs:?}"); - return Ok(()); - } - - info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); - if let Err(err) = self.swarm.dial( - DialOpts::peer_id(peer_id) - .condition(PeerCondition::NotDialing) - .addresses(addrs.iter().cloned().collect()) - .build(), - ) { - warn!(%peer_id, ?addrs, "dialing error: {err:?}"); - } - - trace!( - "SwarmEvent handled in {:?}: {event_string:?}", - start.elapsed() - ); - return Ok(()); - } - - // If we are not local, we care only for peers that we dialed and thus are reachable. - if self.local || has_dialed { - // A bad node cannot establish a connection with us. So we can add it to the RT directly. - - // With the new bootstrap cache, the workload is distributed, - // hence no longer need to replace bootstrap nodes for workload share. - // self.remove_bootstrap_from_full(peer_id); - - // Avoid have `direct link format` addrs co-exists with `relay` addr - if has_relayed { - addrs.retain(|multiaddr| { - multiaddr.iter().any(|p| matches!(p, Protocol::P2pCircuit)) - }); - } - - debug!(%peer_id, ?addrs, "identify: attempting to add addresses to routing table"); - - // Attempt to add the addresses to the routing table. - for multiaddr in addrs { - let _routing_update = self - .swarm - .behaviour_mut() - .kademlia - .add_address(&peer_id, multiaddr); - } - } - trace!( - "SwarmEvent handled in {:?}: {event_string:?}", - start.elapsed() - ); - } - // Log the other Identify events. - libp2p::identify::Event::Sent { .. } => debug!("identify: {iden:?}"), - libp2p::identify::Event::Pushed { .. } => debug!("identify: {iden:?}"), - libp2p::identify::Event::Error { .. } => debug!("identify: {iden:?}"), - } - } - #[cfg(feature = "local")] - SwarmEvent::Behaviour(NodeEvent::Mdns(mdns_event)) => { - event_string = "mdns"; - match *mdns_event { - mdns::Event::Discovered(list) => { - if self.local { - for (peer_id, addr) in list { - // The multiaddr does not contain the peer ID, so add it. - let addr = addr.with(Protocol::P2p(peer_id)); - - info!(%addr, "mDNS node discovered and dialing"); - - if let Err(err) = self.dial(addr.clone()) { - warn!(%addr, "mDNS node dial error: {err:?}"); - } - } - } - } - mdns::Event::Expired(peer) => { - debug!("mdns peer {peer:?} expired"); - } - } + self.handle_identify_event(*event); } SwarmEvent::NewListenAddr { mut address, @@ -331,6 +144,26 @@ impl SwarmDriver { // all addresses are effectively external here... // this is needed for Kad Mode::Server self.swarm.add_external_address(address.clone()); + + // If we are local, add our own address(es) to cache + if let Some(bootstrap_cache) = self.bootstrap_cache.as_mut() { + tracing::info!("Adding listen address to bootstrap cache"); + + let config = bootstrap_cache.config().clone(); + let mut old_cache = bootstrap_cache.clone(); + + if let Ok(new) = BootstrapCacheStore::new(config) { + self.bootstrap_cache = Some(new); + old_cache.add_addr(address.clone()); + + // Save cache to disk. + crate::time::spawn(async move { + if let Err(err) = old_cache.sync_and_flush_to_disk(true) { + error!("Failed to save bootstrap cache: {err}"); + } + }); + } + } } else if let Some(external_add_manager) = self.external_address_manager.as_mut() { @@ -341,6 +174,13 @@ impl SwarmDriver { } } + if tracing::level_enabled!(tracing::Level::DEBUG) { + let all_external_addresses = self.swarm.external_addresses().collect_vec(); + let all_listeners = self.swarm.listeners().collect_vec(); + debug!("All our listeners: {all_listeners:?}"); + debug!("All our external addresses: {all_external_addresses:?}"); + } + self.send_event(NetworkEvent::NewListenAddr(address.clone())); } SwarmEvent::ListenerClosed { @@ -361,6 +201,14 @@ impl SwarmDriver { } => { event_string = "incoming"; debug!("IncomingConnection ({connection_id:?}) with local_addr: {local_addr:?} send_back_addr: {send_back_addr:?}"); + #[cfg(feature = "open-metrics")] + if let Some(relay_manager) = self.relay_manager.as_mut() { + relay_manager.on_incoming_connection( + &connection_id, + &local_addr, + &send_back_addr, + ); + } } SwarmEvent::ConnectionEstablished { peer_id, @@ -378,6 +226,10 @@ impl SwarmDriver { .on_established_incoming_connection(local_addr.clone()); } } + #[cfg(feature = "open-metrics")] + if let Some(relay_manager) = self.relay_manager.as_mut() { + relay_manager.on_connection_established(&peer_id, &connection_id); + } let _ = self.live_connected_peers.insert( connection_id, @@ -424,8 +276,8 @@ impl SwarmDriver { let connection_details = self.live_connected_peers.remove(&connection_id); self.record_connection_metrics(); - // we need to decide if this was a critical error and the peer should be removed from the routing table - let should_clean_peer = match error { + // we need to decide if this was a critical error and if we should report it to the Issue tracker + let is_critical_error = match error { DialError::Transport(errors) => { // as it's an outgoing error, if it's transport based we can assume it is _our_ fault // @@ -434,21 +286,27 @@ impl SwarmDriver { // unless there are _specific_ errors (connection refused eg) error!("Dial errors len : {:?}", errors.len()); let mut there_is_a_serious_issue = false; + // Libp2p throws errors for all the listen addr (including private) of the remote peer even + // though we try to dial just the global/public addr. This would mean that we get + // MultiaddrNotSupported error for the private addr of the peer. + // + // Just a single MultiaddrNotSupported error is not a critical issue, but if all the listen + // addrs of the peer are private, then it is a critical issue. + let mut all_multiaddr_not_supported = true; for (_addr, err) in errors { - error!("OutgoingTransport error : {err:?}"); - match err { TransportError::MultiaddrNotSupported(addr) => { - warn!("Multiaddr not supported : {addr:?}"); + warn!("OutgoingConnectionError: Transport::MultiaddrNotSupported {addr:?}. This can be ignored if the peer has atleast one global address."); #[cfg(feature = "loud")] { - println!("Multiaddr not supported : {addr:?}"); + warn!("OutgoingConnectionError: Transport::MultiaddrNotSupported {addr:?}. This can be ignored if the peer has atleast one global address."); println!("If this was your bootstrap peer, restart your node with a supported multiaddr"); } - // if we can't dial a peer on a given address, we should remove it from the routing table - there_is_a_serious_issue = true } TransportError::Other(err) => { + error!("OutgoingConnectionError: Transport::Other {err:?}"); + + all_multiaddr_not_supported = false; let problematic_errors = [ "ConnectionRefused", "HostUnreachable", @@ -481,6 +339,10 @@ impl SwarmDriver { } } } + if all_multiaddr_not_supported { + warn!("All multiaddrs had MultiaddrNotSupported error for {failed_peer_id:?}. Marking it as a serious issue."); + there_is_a_serious_issue = true; + } there_is_a_serious_issue } DialError::NoAddresses => { @@ -521,26 +383,15 @@ impl SwarmDriver { } }; - if should_clean_peer { - warn!("Tracking issue of {failed_peer_id:?}. Clearing it out for now"); + if is_critical_error { + warn!("Outgoing Connection error to {failed_peer_id:?} is considered as critical. Marking it as an issue."); + self.record_node_issue(failed_peer_id, NodeIssue::ConnectionIssue); - // Just track failures during outgoing connection with `failed_peer_id` inside the bootstrap cache. - // OutgoingConnectionError without peer_id can happen when dialing multiple addresses of a peer. - // And similarly IncomingConnectionError can happen when a peer has multiple transports/listen addrs. if let (Some((_, failed_addr, _)), Some(bootstrap_cache)) = (connection_details, self.bootstrap_cache.as_mut()) { bootstrap_cache.update_addr_status(&failed_addr, false); } - - if let Some(dead_peer) = self - .swarm - .behaviour_mut() - .kademlia - .remove_peer(&failed_peer_id) - { - self.update_on_peer_removal(*dead_peer.node.key.preimage()); - } } } SwarmEvent::IncomingConnectionError { @@ -550,7 +401,7 @@ impl SwarmDriver { error, } => { event_string = "Incoming ConnErr"; - // Only log as ERROR if the the connection is not adjacent to an already established connection id from + // Only log as ERROR if the connection is not adjacent to an already established connection id from // the same IP address. // // If a peer contains multiple transports/listen addrs, we might try to open multiple connections, @@ -561,15 +412,24 @@ impl SwarmDriver { // giving time for ConnectionEstablished to be hopefully processed. // And since we don't do anything critical with this event, the order and time of processing is // not critical. - if self.should_we_log_incoming_connection_error(connection_id, &send_back_addr) { - error!("IncomingConnectionError from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); + if self.is_incoming_connection_error_valid(connection_id, &send_back_addr) { + error!("IncomingConnectionError Valid from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); + + // This is best approximation that we can do to prevent harmless errors from affecting the external + // address health. + if let Some(external_addr_manager) = self.external_address_manager.as_mut() { + external_addr_manager + .on_incoming_connection_error(local_addr.clone(), &mut self.swarm); + } } else { - debug!("IncomingConnectionError from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); + debug!("IncomingConnectionError InValid from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); } - if let Some(external_addr_manager) = self.external_address_manager.as_mut() { - external_addr_manager - .on_incoming_connection_error(local_addr.clone(), &mut self.swarm); + + #[cfg(feature = "open-metrics")] + if let Some(relay_manager) = self.relay_manager.as_mut() { + relay_manager.on_incomming_connection_error(&send_back_addr, &connection_id); } + let _ = self.live_connected_peers.remove(&connection_id); self.record_connection_metrics(); } @@ -773,7 +633,7 @@ impl SwarmDriver { } // Do not log IncomingConnectionError if the ConnectionId is adjacent to an already established connection. - fn should_we_log_incoming_connection_error(&self, id: ConnectionId, addr: &Multiaddr) -> bool { + fn is_incoming_connection_error_valid(&self, id: ConnectionId, addr: &Multiaddr) -> bool { let Ok(id) = format!("{id}").parse::() else { return true; }; diff --git a/ant-networking/src/external_address.rs b/ant-networking/src/external_address.rs index ad71dd2c16..0491f90ad9 100644 --- a/ant-networking/src/external_address.rs +++ b/ant-networking/src/external_address.rs @@ -372,7 +372,9 @@ impl ExternalAddressManager { if !removed_confirmed.is_empty() { info!("Removed external addresses due to connection errors on port {port}: {removed_confirmed:?}"); } - Self::print_swarm_state(swarm); + if !removed_candidates.is_empty() || !removed_confirmed.is_empty() { + Self::print_swarm_state(swarm); + } } } @@ -483,9 +485,9 @@ impl ExternalAddressManager { fn print_swarm_state(swarm: &Swarm) { let listen_addr = swarm.listeners().collect::>(); - info!("All Listen addresses: {listen_addr:?}"); let external_addr = swarm.external_addresses().collect::>(); - info!("All External addresses: {external_addr:?}"); + // Combined output to reduce cpu/disk usage. + info!("All External addresses: {external_addr:?}, and listen addresses: {listen_addr:?}"); } } diff --git a/ant-networking/src/graph.rs b/ant-networking/src/graph.rs new file mode 100644 index 0000000000..132eeac9fc --- /dev/null +++ b/ant-networking/src/graph.rs @@ -0,0 +1,31 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{NetworkError, Result}; +use ant_protocol::storage::{DataTypes, GraphEntry}; +use ant_protocol::{ + storage::{try_deserialize_record, RecordHeader, RecordKind}, + PrettyPrintRecordKey, +}; +use libp2p::kad::Record; + +pub fn get_graph_entry_from_record(record: &Record) -> Result> { + let header = RecordHeader::from_record(record)?; + if let RecordKind::DataOnly(DataTypes::GraphEntry) = header.kind { + let entry = try_deserialize_record::>(record)?; + Ok(entry) + } else { + warn!( + "RecordKind mismatch while trying to retrieve graph_entry from record {:?}", + PrettyPrintRecordKey::from(&record.key) + ); + Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly( + DataTypes::GraphEntry, + ))) + } +} diff --git a/ant-networking/src/lib.rs b/ant-networking/src/lib.rs index fca47f18d0..8abaaf9ead 100644 --- a/ant-networking/src/lib.rs +++ b/ant-networking/src/lib.rs @@ -12,11 +12,13 @@ extern crate tracing; mod bootstrap; mod circular_vec; mod cmd; +mod config; mod driver; mod error; mod event; mod external_address; mod fifo_register; +mod graph; mod log_markers; #[cfg(feature = "open-metrics")] mod metrics; @@ -25,8 +27,7 @@ mod record_store; mod record_store_api; mod relay_manager; mod replication_fetcher; -pub mod target_arch; -mod transactions; +pub mod time; mod transport; use cmd::LocalSwarmCmd; @@ -35,30 +36,29 @@ use xor_name::XorName; // re-export arch dependent deps for use in the crate, or above pub use self::{ cmd::{NodeIssue, SwarmLocalState}, - driver::{ - GetRecordCfg, NetworkBuilder, PutRecordCfg, SwarmDriver, VerificationKind, MAX_PACKET_SIZE, - }, + config::{GetRecordCfg, PutRecordCfg, ResponseQuorum, RetryStrategy, VerificationKind}, + driver::{NetworkBuilder, SwarmDriver, MAX_PACKET_SIZE}, error::{GetRecordError, NetworkError}, event::{MsgResponder, NetworkEvent}, + graph::get_graph_entry_from_record, record_store::NodeRecordStore, - transactions::get_transactions_from_record, }; #[cfg(feature = "open-metrics")] pub use metrics::service::MetricsRegistries; -pub use target_arch::{interval, sleep, spawn, Instant, Interval}; +pub use time::{interval, sleep, spawn, Instant, Interval}; use self::{cmd::NetworkSwarmCmd, error::Result}; use ant_evm::{PaymentQuote, QuotingMetrics}; use ant_protocol::{ error::Error as ProtocolError, messages::{ChunkProof, Nonce, Query, QueryResponse, Request, Response}, - storage::{RecordType, RetryStrategy, Scratchpad}, + storage::{DataTypes, Pointer, Scratchpad, ValidationType}, NetworkAddress, PrettyPrintKBucketKey, PrettyPrintRecordKey, CLOSE_GROUP_SIZE, }; use futures::future::select_all; use libp2p::{ identity::Keypair, - kad::{KBucketDistance, KBucketKey, Quorum, Record, RecordKey}, + kad::{KBucketDistance, KBucketKey, Record, RecordKey}, multiaddr::Protocol, request_response::OutboundFailure, Multiaddr, PeerId, @@ -75,11 +75,10 @@ use tokio::sync::{ }; use tokio::time::Duration; use { - ant_protocol::storage::Transaction, + ant_protocol::storage::GraphEntry, ant_protocol::storage::{ try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, }, - ant_registers::SignedRegister, std::collections::HashSet, }; @@ -276,9 +275,11 @@ impl Network { let (sender, receiver) = oneshot::channel(); self.send_local_swarm_cmd(LocalSwarmCmd::GetReplicateCandidates { data_addr, sender }); - receiver + let candidate = receiver .await - .map_err(|_e| NetworkError::InternalMsgChannelDropped) + .map_err(|_e| NetworkError::InternalMsgChannelDropped)??; + + Ok(candidate) } /// Get the Chunk existence proof from the close nodes to the provided chunk address. @@ -288,15 +289,13 @@ impl Network { chunk_address: NetworkAddress, nonce: Nonce, expected_proof: ChunkProof, - quorum: Quorum, - retry_strategy: Option, + quorum: ResponseQuorum, + retry_strategy: RetryStrategy, ) -> Result<()> { - let total_attempts = retry_strategy - .map(|strategy| strategy.attempts()) - .unwrap_or(1); + let total_attempts = retry_strategy.attempts(); let pretty_key = PrettyPrintRecordKey::from(&chunk_address.to_record_key()).into_owned(); - let expected_n_verified = get_quorum_value(&quorum); + let expected_n_verified = quorum.get_value(); let mut close_nodes = Vec::new(); let mut retry_attempts = 0; @@ -378,6 +377,8 @@ impl Network { pub async fn get_store_quote_from_network( &self, record_address: NetworkAddress, + data_type: u32, + data_size: usize, ignore_peers: Vec, ) -> Result> { // The requirement of having at least CLOSE_GROUP_SIZE @@ -394,12 +395,14 @@ impl Network { if close_nodes.is_empty() { error!("Can't get store_cost of {record_address:?}, as all close_nodes are ignored"); - return Err(NetworkError::NoStoreCostResponses); + return Err(NetworkError::NotEnoughPeersForStoreCostRequest); } // Client shall decide whether to carry out storage verification or not. let request = Request::Query(Query::GetStoreQuote { key: record_address.clone(), + data_type, + data_size, nonce: None, difficulty: 0, }); @@ -411,6 +414,8 @@ impl Network { let mut peer_already_have_it = 0; let enough_peers_already_have_it = close_nodes.len() / 2; + let mut peers_returned_error = 0; + // loop over responses let mut all_quotes = vec![]; let mut quotes_to_pay = vec![]; @@ -425,12 +430,19 @@ impl Network { if !storage_proofs.is_empty() { debug!("Storage proofing during GetStoreQuote to be implemented."); } + // Check the quote itself is valid. if !quote.check_is_signed_by_claimed_peer(peer) { warn!("Received invalid quote from {peer_address:?}, {quote:?}"); continue; } + // Check if the returned data type matches the request + if quote.quoting_metrics.data_type != data_type { + warn!("Received invalid quote from {peer_address:?}, {quote:?}. Data type did not match the request."); + continue; + } + all_quotes.push((peer_address.clone(), quote.clone())); quotes_to_pay.push((peer, quote)); } @@ -451,66 +463,24 @@ impl Network { } Err(err) => { error!("Got an error while requesting quote from peer {peer:?}: {err}"); + peers_returned_error += 1; } _ => { error!("Got an unexpected response while requesting quote from peer {peer:?}: {response:?}"); + peers_returned_error += 1; } } } - Ok(quotes_to_pay) - } - - /// Get register from network. - /// Due to the nature of the p2p network, it's not guaranteed there is only one version - /// exists in the network all the time. - /// The scattering of the register will be more like `ring layered`. - /// Meanwhile, `kad::get_record` will terminate with first majority copies returned, - /// which has the risk of returning with old versions. - /// So, to improve the accuracy, query closest_peers first, then fetch registers - /// And merge them if they are with different content. - pub async fn get_register_record_from_network( - &self, - key: RecordKey, - ) -> Result> { - let record_address = NetworkAddress::from_record_key(&key); - // The requirement of having at least CLOSE_GROUP_SIZE - // close nodes will be checked internally automatically. - let close_nodes = self - .client_get_all_close_peers_in_range_or_close_group(&record_address) - .await?; - - let self_address = NetworkAddress::from_peer(self.peer_id()); - let request = Request::Query(Query::GetRegisterRecord { - requester: self_address, - key: record_address.clone(), - }); - let responses = self - .send_and_get_responses(&close_nodes, &request, true) - .await; - - // loop over responses, collecting all fetched register records - let mut all_register_copies = HashMap::new(); - for response in responses.into_values().flatten() { - match response { - Response::Query(QueryResponse::GetRegisterRecord(Ok((holder, content)))) => { - let register_record = Record::new(key.clone(), content.to_vec()); - let content_hash = XorName::from_content(®ister_record.value); - debug!( - "RegisterRecordReq of {record_address:?} received register of version {content_hash:?} from {holder:?}" - ); - let _ = all_register_copies.insert(content_hash, register_record); - } - _ => { - error!( - "RegisterRecordReq of {record_address:?} received error response, was {:?}", - response - ); - } - } + if quotes_to_pay.is_empty() { + error!( + "Could not fetch any quotes. {} peers returned an error.", + peers_returned_error + ); + return Err(NetworkError::NoStoreCostResponses); } - Ok(all_register_copies) + Ok(quotes_to_pay) } /// Get the Record from the network @@ -518,20 +488,14 @@ impl Network { /// In case a target_record is provided, only return when fetched target. /// Otherwise count it as a failure when all attempts completed. /// - /// It also handles the split record error for transactions and registers. - /// For transactions, it accumulates the transactions and returns an error if more than one. - /// For registers, it merges the registers and returns the merged record. + /// It also handles the split record error for GraphEntry. pub async fn get_record_from_network( &self, key: RecordKey, cfg: &GetRecordCfg, ) -> Result { let pretty_key = PrettyPrintRecordKey::from(&key); - let mut backoff = cfg - .retry_strategy - .unwrap_or(RetryStrategy::None) - .backoff() - .into_iter(); + let mut backoff = cfg.retry_strategy.backoff().into_iter(); loop { info!("Getting record from network of {pretty_key:?}. with cfg {cfg:?}",); @@ -581,7 +545,7 @@ impl Network { GetRecordError::SplitRecord { result_map } => { error!("Encountered a split record for {pretty_key:?}."); if let Some(record) = Self::handle_split_record_error(result_map, &key)? { - info!("Merged the split record (register) for {pretty_key:?}, into a single record"); + info!("Merged the split record for {pretty_key:?}, into a single record"); return Ok(record); } } @@ -592,7 +556,7 @@ impl Network { match backoff.next() { Some(Some(duration)) => { - crate::target_arch::sleep(duration).await; + crate::time::sleep(duration).await; debug!("Getting record from network of {pretty_key:?} via backoff..."); } _ => break Err(err.into()), @@ -601,19 +565,17 @@ impl Network { } /// Handle the split record error. - /// Transaction: Accumulate transactions. - /// Register: Merge registers and return the merged record. fn handle_split_record_error( result_map: &HashMap)>, key: &RecordKey, ) -> std::result::Result, NetworkError> { let pretty_key = PrettyPrintRecordKey::from(key); - // attempt to deserialise and accumulate any transactions or registers + // attempt to deserialise and accumulate all GraphEntries let results_count = result_map.len(); - let mut accumulated_transactions = HashSet::new(); - let mut collected_registers = Vec::new(); + let mut accumulated_graphentries = HashSet::new(); let mut valid_scratchpad: Option = None; + let mut valid_pointer: Option = None; if results_count > 1 { let mut record_kind = None; @@ -631,49 +593,45 @@ impl Network { } match kind { - RecordKind::Chunk - | RecordKind::ChunkWithPayment - | RecordKind::TransactionWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::ScratchpadWithPayment => { + RecordKind::DataOnly(DataTypes::Chunk) | RecordKind::DataWithPayment(_) => { error!("Encountered a split record for {pretty_key:?} with unexpected RecordKind {kind:?}, skipping."); continue; } - RecordKind::Transaction => { - info!("For record {pretty_key:?}, we have a split record for a transaction attempt. Accumulating transactions"); - - match get_transactions_from_record(record) { - Ok(transactions) => { - accumulated_transactions.extend(transactions); + RecordKind::DataOnly(DataTypes::GraphEntry) => { + match get_graph_entry_from_record(record) { + Ok(graphentries) => { + accumulated_graphentries.extend(graphentries); + info!("For record {pretty_key:?}, we have a split record for a GraphEntry. Accumulating GraphEntry: {}", accumulated_graphentries.len()); } Err(_) => { + warn!("Failed to deserialize GraphEntry for {pretty_key:?}, skipping accumulation"); continue; } } } - RecordKind::Register => { - info!("For record {pretty_key:?}, we have a split record for a register. Accumulating registers"); - let Ok(register) = try_deserialize_record::(record) else { + RecordKind::DataOnly(DataTypes::Pointer) => { + info!("For record {pretty_key:?}, we have a split record for a pointer. Selecting the one with the highest count"); + let Ok(pointer) = try_deserialize_record::(record) else { error!( - "Failed to deserialize register {pretty_key}. Skipping accumulation" + "Failed to deserialize pointer {pretty_key}. Skipping accumulation" ); continue; }; - match register.verify() { - Ok(_) => { - collected_registers.push(register); - } - Err(_) => { - error!( - "Failed to verify register for {pretty_key} at address: {}. Skipping accumulation", - register.address() - ); + if !pointer.verify_signature() { + warn!("Rejecting Pointer for {pretty_key} PUT with invalid signature"); + continue; + } + + if let Some(old) = &valid_pointer { + if old.counter() >= pointer.counter() { + info!("Rejecting Pointer for {pretty_key} with lower count than the previous one"); continue; } } + valid_pointer = Some(pointer); } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { info!("For record {pretty_key:?}, we have a split record for a scratchpad. Selecting the one with the highest count"); let Ok(scratchpad) = try_deserialize_record::(record) else { error!( @@ -682,42 +640,37 @@ impl Network { continue; }; - if !scratchpad.is_valid() { + if !scratchpad.verify_signature() { warn!( - "Rejecting Scratchpad for {pretty_key} PUT with invalid signature during split record error" + "Rejecting Scratchpad for {pretty_key} PUT with invalid signature" ); continue; } if let Some(old) = &valid_scratchpad { - if old.count() >= scratchpad.count() { - info!( - "Rejecting Scratchpad for {pretty_key} with lower count than the previous one" - ); + if old.counter() >= scratchpad.counter() { + info!("Rejecting Scratchpad for {pretty_key} with lower count than the previous one"); continue; - } else { - valid_scratchpad = Some(scratchpad); } - } else { - valid_scratchpad = Some(scratchpad); } + valid_scratchpad = Some(scratchpad); } } } } - // Return the accumulated transactions as a single record - if accumulated_transactions.len() > 1 { - info!("For record {pretty_key:?} task found split record for a transaction, accumulated and sending them as a single record"); - let accumulated_transactions = accumulated_transactions + // Return the accumulated GraphEntries as a single record + if accumulated_graphentries.len() > 1 { + info!("For record {pretty_key:?} task found split record for a GraphEntry, accumulated and sending them as a single record"); + let accumulated_graphentries = accumulated_graphentries .into_iter() - .collect::>(); + .collect::>(); let record = Record { key: key.clone(), - value: try_serialize_record(&accumulated_transactions, RecordKind::Transaction) + value: try_serialize_record(&accumulated_graphentries, RecordKind::DataOnly(DataTypes::GraphEntry)) .map_err(|err| { error!( - "Error while serializing the accumulated transactions for {pretty_key:?}: {err:?}" + "Error while serializing the accumulated GraphEntries for {pretty_key:?}: {err:?}" ); NetworkError::from(err) })? @@ -726,23 +679,15 @@ impl Network { expires: None, }; return Ok(Some(record)); - } else if !collected_registers.is_empty() { - info!("For record {pretty_key:?} task found multiple registers, merging them."); - let signed_register = collected_registers.iter().fold(collected_registers[0].clone(), |mut acc, x| { - if let Err(e) = acc.merge(x) { - warn!("Ignoring forked register as we failed to merge conflicting registers at {}: {e}", x.address()); - } - acc - }); - - let record_value = try_serialize_record(&signed_register, RecordKind::Register) - .map_err(|err| { - error!( - "Error while serializing the merged register for {pretty_key:?}: {err:?}" - ); - NetworkError::from(err) - })? - .to_vec(); + } else if let Some(pointer) = valid_pointer { + info!("For record {pretty_key:?} task found a valid pointer, returning it."); + let record_value = + try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer)) + .map_err(|err| { + error!("Error while serializing the pointer for {pretty_key:?}: {err:?}"); + NetworkError::from(err) + })? + .to_vec(); let record = Record { key: key.clone(), @@ -752,17 +697,20 @@ impl Network { }; return Ok(Some(record)); } else if let Some(scratchpad) = valid_scratchpad { - info!("Found a valid scratchpad for {pretty_key:?}, returning it"); - let record = Record { - key: key.clone(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad) + info!("For record {pretty_key:?} task found a valid scratchpad, returning it."); + let record_value = + try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad)) .map_err(|err| { error!( - "Error while serializing valid scratchpad for {pretty_key:?}: {err:?}" + "Error while serializing the scratchpad for {pretty_key:?}: {err:?}" ); NetworkError::from(err) })? - .to_vec(), + .to_vec(); + + let record = Record { + key: key.clone(), + value: record_value, publisher: None, expires: None, }; @@ -775,13 +723,21 @@ impl Network { pub async fn get_local_quoting_metrics( &self, key: RecordKey, + data_type: u32, + data_size: usize, ) -> Result<(QuotingMetrics, bool)> { let (sender, receiver) = oneshot::channel(); - self.send_local_swarm_cmd(LocalSwarmCmd::GetLocalQuotingMetrics { key, sender }); + self.send_local_swarm_cmd(LocalSwarmCmd::GetLocalQuotingMetrics { + key, + data_type, + data_size, + sender, + }); - receiver + let quoting_metrics = receiver .await - .map_err(|_e| NetworkError::InternalMsgChannelDropped) + .map_err(|_e| NetworkError::InternalMsgChannelDropped)??; + Ok(quoting_metrics) } /// Notify the node receicced a payment. @@ -817,11 +773,7 @@ impl Network { /// If verify is on, we retry. pub async fn put_record(&self, record: Record, cfg: &PutRecordCfg) -> Result<()> { let pretty_key = PrettyPrintRecordKey::from(&record.key); - let mut backoff = cfg - .retry_strategy - .unwrap_or(RetryStrategy::None) - .backoff() - .into_iter(); + let mut backoff = cfg.retry_strategy.backoff().into_iter(); loop { info!( @@ -833,12 +785,12 @@ impl Network { Err(err) => err, }; - // FIXME: Skip if we get a permanent error during verification, e.g., DoubleSpendAttempt + // FIXME: Skip if we get a permanent error during verification warn!("Failed to PUT record with key: {pretty_key:?} to network (retry via backoff) with error: {err:?}"); match backoff.next() { Some(Some(duration)) => { - crate::target_arch::sleep(duration).await; + crate::time::sleep(duration).await; } _ => break Err(err), } @@ -928,20 +880,23 @@ impl Network { } /// Notify ReplicationFetch a fetch attempt is completed. - /// (but it won't trigger any real writes to disk, say fetched an old version of register) - pub fn notify_fetch_completed(&self, key: RecordKey, record_type: RecordType) { + /// (but it won't trigger any real writes to disk) + pub fn notify_fetch_completed(&self, key: RecordKey, record_type: ValidationType) { self.send_local_swarm_cmd(LocalSwarmCmd::FetchCompleted((key, record_type))) } /// Put `Record` to the local RecordStore /// Must be called after the validations are performed on the Record - pub fn put_local_record(&self, record: Record) { + pub fn put_local_record(&self, record: Record, is_client_put: bool) { debug!( "Writing Record locally, for {:?} - length {:?}", PrettyPrintRecordKey::from(&record.key), record.value.len() ); - self.send_local_swarm_cmd(LocalSwarmCmd::PutLocalRecord { record }) + self.send_local_swarm_cmd(LocalSwarmCmd::PutLocalRecord { + record, + is_client_put, + }) } /// Returns true if a RecordKey is present locally in the RecordStore @@ -952,21 +907,24 @@ impl Network { sender, }); - receiver + let is_present = receiver .await - .map_err(|_e| NetworkError::InternalMsgChannelDropped) + .map_err(|_e| NetworkError::InternalMsgChannelDropped)??; + + Ok(is_present) } /// Returns the Addresses of all the locally stored Records pub async fn get_all_local_record_addresses( &self, - ) -> Result> { + ) -> Result> { let (sender, receiver) = oneshot::channel(); self.send_local_swarm_cmd(LocalSwarmCmd::GetAllLocalRecordAddresses { sender }); - receiver + let addrs = receiver .await - .map_err(|_e| NetworkError::InternalMsgChannelDropped) + .map_err(|_e| NetworkError::InternalMsgChannelDropped)??; + Ok(addrs) } /// Send `Request` to the given `PeerId` and await for the response. If `self` is the recipient, @@ -1042,6 +1000,14 @@ impl Network { self.send_local_swarm_cmd(LocalSwarmCmd::TriggerIntervalReplication) } + pub fn add_fresh_records_to_the_replication_fetcher( + &self, + holder: NetworkAddress, + keys: Vec<(NetworkAddress, ValidationType)>, + ) { + self.send_local_swarm_cmd(LocalSwarmCmd::AddFreshReplicateRecords { holder, keys }) + } + pub fn record_node_issues(&self, peer_id: PeerId, issue: NodeIssue) { self.send_local_swarm_cmd(LocalSwarmCmd::RecordNodeIssue { peer_id, issue }); } @@ -1058,6 +1024,10 @@ impl Network { self.send_local_swarm_cmd(LocalSwarmCmd::AddNetworkDensitySample { distance }) } + pub fn notify_peer_scores(&self, peer_scores: Vec<(PeerId, bool)>) { + self.send_local_swarm_cmd(LocalSwarmCmd::NotifyPeerScores { peer_scores }) + } + /// Helper to send NetworkSwarmCmd fn send_network_swarm_cmd(&self, cmd: NetworkSwarmCmd) { send_network_swarm_cmd(self.network_swarm_cmd_sender().clone(), cmd); @@ -1161,16 +1131,6 @@ impl Network { } } -/// Get the value of the provided Quorum -pub fn get_quorum_value(quorum: &Quorum) -> usize { - match quorum { - Quorum::Majority => close_group_majority(), - Quorum::All => CLOSE_GROUP_SIZE, - Quorum::N(v) => v.get(), - Quorum::One => 1, - } -} - /// Verifies if `Multiaddr` contains IPv4 address that is not global. /// This is used to filter out unroutable addresses from the Kademlia routing table. pub fn multiaddr_is_global(multiaddr: &Multiaddr) -> bool { @@ -1287,10 +1247,10 @@ pub(crate) fn send_network_swarm_cmd( mod tests { use super::*; - #[test] - fn test_network_sign_verify() -> eyre::Result<()> { + #[tokio::test] + async fn test_network_sign_verify() -> eyre::Result<()> { let (network, _, _) = - NetworkBuilder::new(Keypair::generate_ed25519(), false).build_client()?; + NetworkBuilder::new(Keypair::generate_ed25519(), false).build_client(); let msg = b"test message"; let sig = network.sign(msg)?; assert!(network.verify(msg, &sig)); diff --git a/ant-networking/src/metrics/bad_node.rs b/ant-networking/src/metrics/bad_node.rs index 4e85931126..23d8a8afa1 100644 --- a/ant-networking/src/metrics/bad_node.rs +++ b/ant-networking/src/metrics/bad_node.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::target_arch::interval; +use crate::time::interval; use ant_protocol::CLOSE_GROUP_SIZE; use libp2p::PeerId; use prometheus_client::{ @@ -46,7 +46,7 @@ struct ShunnedByCloseGroup { old_new_group_shunned_list: HashSet, } -/// A struct to record the the number of reports against our node across different time frames. +/// A struct to record the number of reports against our node across different time frames. struct ShunnedCountAcrossTimeFrames { metric: Family, shunned_report_tracker: Vec, diff --git a/ant-networking/src/metrics/mod.rs b/ant-networking/src/metrics/mod.rs index 1efc7a2334..c78508ecb1 100644 --- a/ant-networking/src/metrics/mod.rs +++ b/ant-networking/src/metrics/mod.rs @@ -8,14 +8,14 @@ // Implementation to record `libp2p::upnp::Event` metrics mod bad_node; +mod relay_client; pub mod service; -#[cfg(feature = "upnp")] mod upnp; use std::sync::atomic::AtomicU64; use crate::MetricsRegistries; -use crate::{log_markers::Marker, target_arch::sleep}; +use crate::{log_markers::Marker, time::sleep}; use bad_node::{BadNodeMetrics, BadNodeMetricsMsg, TimeFrame}; use libp2p::{ metrics::{Metrics as Libp2pMetrics, Recorder}, @@ -37,8 +37,8 @@ pub(crate) struct NetworkMetricsRecorder { // Must directly call self.libp2p_metrics.record(libp2p_event) with Recorder trait in scope. But since we have // re-implemented the trait for the wrapper struct, we can instead call self.record(libp2p_event) libp2p_metrics: Libp2pMetrics, - #[cfg(feature = "upnp")] upnp_events: Family, + relay_client_events: Family, // metrics from ant-networking pub(crate) connected_peers: Gauge, @@ -46,6 +46,7 @@ pub(crate) struct NetworkMetricsRecorder { pub(crate) open_connections: Gauge, pub(crate) peers_in_routing_table: Gauge, pub(crate) records_stored: Gauge, + pub(crate) relay_reservation_health: Gauge, // quoting metrics relevant_records: Gauge, @@ -86,6 +87,12 @@ impl NetworkMetricsRecorder { "The number of records stored locally", records_stored.clone(), ); + let relay_reservation_health = Gauge::::default(); + sub_registry.register( + "relay_reservation_health", + "The average health of all the relay reservation connections. Value is between 0-1", + relay_reservation_health.clone(), + ); let connected_peers = Gauge::default(); sub_registry.register( @@ -127,15 +134,20 @@ impl NetworkMetricsRecorder { bad_peers_count.clone(), ); - #[cfg(feature = "upnp")] let upnp_events = Family::default(); - #[cfg(feature = "upnp")] sub_registry.register( "upnp_events", "Events emitted by the UPnP behaviour", upnp_events.clone(), ); + let relay_client_events = Family::default(); + sub_registry.register( + "relay_client_events", + "Events emitted by the relay client", + relay_client_events.clone(), + ); + let process_memory_used_mb = Gauge::::default(); sub_registry.register( "process_memory_used_mb", @@ -209,13 +221,14 @@ impl NetworkMetricsRecorder { ); let network_metrics = Self { libp2p_metrics, - #[cfg(feature = "upnp")] upnp_events, + relay_client_events, records_stored, estimated_network_size, connected_peers, open_connections, + relay_reservation_health, peers_in_routing_table, relevant_records, max_records, @@ -279,7 +292,7 @@ impl NetworkMetricsRecorder { let _ = self.shunned_count.inc(); let bad_nodes_notifier = self.bad_nodes_notifier.clone(); let flagged_by = *flagged_by; - crate::target_arch::spawn(async move { + crate::time::spawn(async move { if let Err(err) = bad_nodes_notifier .send(BadNodeMetricsMsg::ShunnedByPeer(flagged_by)) .await @@ -310,7 +323,7 @@ impl NetworkMetricsRecorder { pub(crate) fn record_change_in_close_group(&self, new_close_group: Vec) { let bad_nodes_notifier = self.bad_nodes_notifier.clone(); - crate::target_arch::spawn(async move { + crate::time::spawn(async move { if let Err(err) = bad_nodes_notifier .send(BadNodeMetricsMsg::CloseGroupUpdated(new_close_group)) .await diff --git a/ant-networking/src/metrics/relay_client.rs b/ant-networking/src/metrics/relay_client.rs new file mode 100644 index 0000000000..cd03822619 --- /dev/null +++ b/ant-networking/src/metrics/relay_client.rs @@ -0,0 +1,47 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue}; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, EncodeLabelSet)] +pub(crate) struct RelayClientEventLabels { + event: EventType, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq, EncodeLabelValue)] +enum EventType { + ReservationReqAccepted, + OutboundCircuitEstablished, + InboundCircuitEstablished, +} + +impl From<&libp2p::relay::client::Event> for EventType { + fn from(event: &libp2p::relay::client::Event) -> Self { + match event { + libp2p::relay::client::Event::ReservationReqAccepted { .. } => { + EventType::ReservationReqAccepted + } + libp2p::relay::client::Event::OutboundCircuitEstablished { .. } => { + EventType::OutboundCircuitEstablished + } + libp2p::relay::client::Event::InboundCircuitEstablished { .. } => { + EventType::InboundCircuitEstablished + } + } + } +} + +impl super::Recorder for super::NetworkMetricsRecorder { + fn record(&self, event: &libp2p::relay::client::Event) { + self.relay_client_events + .get_or_create(&RelayClientEventLabels { + event: event.into(), + }) + .inc(); + } +} diff --git a/ant-networking/src/network_discovery.rs b/ant-networking/src/network_discovery.rs index c0932b6c2f..39fa45e51c 100644 --- a/ant-networking/src/network_discovery.rs +++ b/ant-networking/src/network_discovery.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::target_arch::Instant; +use crate::time::Instant; use ant_protocol::NetworkAddress; use libp2p::{kad::KBucketKey, PeerId}; use rand::{thread_rng, Rng}; diff --git a/ant-networking/src/record_store.rs b/ant-networking/src/record_store.rs index b4ab4ff6b3..ea9222eb73 100644 --- a/ant-networking/src/record_store.rs +++ b/ant-networking/src/record_store.rs @@ -10,16 +10,15 @@ use crate::cmd::LocalSwarmCmd; use crate::driver::MAX_PACKET_SIZE; use crate::send_local_swarm_cmd; -use crate::target_arch::{spawn, Instant}; +use crate::time::{spawn, Instant}; use crate::{event::NetworkEvent, log_markers::Marker}; use aes_gcm_siv::{ aead::{Aead, KeyInit}, Aes256GcmSiv, Key as AesKey, Nonce, }; -use ant_evm::{QuotingMetrics, U256}; +use ant_evm::QuotingMetrics; use ant_protocol::{ - convert_distance_to_u256, - storage::{RecordHeader, RecordKind, RecordType}, + storage::{DataTypes, RecordHeader, RecordKind, ValidationType}, NetworkAddress, PrettyPrintRecordKey, }; use hkdf::Hkdf; @@ -44,13 +43,13 @@ use std::{ time::SystemTime, vec, }; -use tokio::sync::mpsc; +use tokio::{sync::mpsc, time::Duration}; use walkdir::{DirEntry, WalkDir}; use xor_name::XorName; -// A transaction record is at the size of 4KB roughly. +// A GraphEntry record is at the size of 4KB roughly. // Given chunk record is maxed at size of 4MB. -// During Beta phase, it's almost one transaction per chunk, +// During Beta phase, it's almost one GraphEntry per chunk, // which makes the average record size is around 2MB. // Given we are targeting node size to be 32GB, // this shall allow around 16K records. @@ -62,6 +61,10 @@ const MAX_RECORDS_CACHE_SIZE: usize = 25; /// File name of the recorded historical quoting metrics. const HISTORICAL_QUOTING_METRICS_FILENAME: &str = "historic_quoting_metrics"; +/// Defines when the entries inside the cache shall be pruned to free space up. +/// Shall be two times of the PERIODIC_REPLICATION_INTERVAL_MAX_S +const CACHE_TIMEOUT: Duration = Duration::from_secs(360); + fn derive_aes256gcm_siv_from_seed(seed: &[u8; 16]) -> (Aes256GcmSiv, [u8; 4]) { // shall be unique for purpose. let salt = b"autonomi_record_store"; @@ -87,13 +90,15 @@ fn derive_aes256gcm_siv_from_seed(seed: &[u8; 16]) -> (Aes256GcmSiv, [u8; 4]) { struct RecordCache { records_cache: HashMap, cache_size: usize, + cache_timeout: Duration, } impl RecordCache { - fn new(cache_size: usize) -> Self { + fn new(cache_size: usize, cache_timeout: Duration) -> Self { RecordCache { records_cache: HashMap::new(), cache_size, + cache_timeout, } } @@ -108,26 +113,36 @@ impl RecordCache { fn push_back(&mut self, key: Key, record: Record) { self.free_up_space(); - let _ = self.records_cache.insert(key, (record, SystemTime::now())); + let _ = self + .records_cache + .insert(key, (record, SystemTime::now() + self.cache_timeout)); } fn free_up_space(&mut self) { + let current = SystemTime::now(); + // Remove outdated entries first + self.records_cache + .retain(|_key, (_record, timestamp)| *timestamp > current); + while self.records_cache.len() >= self.cache_size { self.remove_oldest_entry() } } fn remove_oldest_entry(&mut self) { - let mut oldest_timestamp = SystemTime::now(); + let mut oldest_timestamp = SystemTime::now() + self.cache_timeout; + let mut key_to_remove = None; - for (_record, timestamp) in self.records_cache.values() { + for (key, (_record, timestamp)) in self.records_cache.iter() { if *timestamp < oldest_timestamp { oldest_timestamp = *timestamp; + key_to_remove = Some(key.clone()); } } - self.records_cache - .retain(|_key, (_record, timestamp)| *timestamp != oldest_timestamp); + if let Some(key) = key_to_remove { + let _ = self.records_cache.remove(&key); + } } } @@ -138,9 +153,9 @@ pub struct NodeRecordStore { /// The configuration of the store. config: NodeRecordStoreConfig, /// Main records store remains unchanged for compatibility - records: HashMap, + records: HashMap, /// Additional index organizing records by distance - records_by_distance: BTreeMap, + records_by_distance: BTreeMap, /// FIFO simple cache of records to reduce read times records_cache: RecordCache, /// Send network events to the node layer. @@ -150,7 +165,7 @@ pub struct NodeRecordStore { /// ilog2 distance range of responsible records /// AKA: how many buckets of data do we consider "close" /// None means accept all records. - responsible_distance_range: Option, + responsible_distance_range: Option, #[cfg(feature = "open-metrics")] /// Used to report the number of records held by the store to the metrics server. record_count_metric: Option, @@ -218,7 +233,7 @@ impl NodeRecordStore { fn update_records_from_an_existing_store( config: &NodeRecordStoreConfig, encryption_details: &(Aes256GcmSiv, [u8; 4]), - ) -> HashMap { + ) -> HashMap { let process_entry = |entry: &DirEntry| -> _ { let path = entry.path(); if path.is_file() { @@ -269,11 +284,19 @@ impl NodeRecordStore { } }; - let record_type = match RecordHeader::is_record_of_type_chunk(&record) { - Ok(true) => RecordType::Chunk, - Ok(false) => { - let xorname_hash = XorName::from_content(&record.value); - RecordType::NonChunk(xorname_hash) + match RecordHeader::get_data_type(&record) { + Ok(data_type) => { + let validate_type = match data_type { + DataTypes::Chunk => ValidationType::Chunk, + _ => { + let xorname_hash = XorName::from_content(&record.value); + ValidationType::NonChunk(xorname_hash) + } + }; + + let address = NetworkAddress::from_record_key(&key); + info!("Existing record {address:?} loaded from: {path:?}"); + return Some((key, (address, validate_type, data_type))); } Err(error) => { warn!( @@ -290,11 +313,7 @@ impl NodeRecordStore { } return None; } - }; - - let address = NetworkAddress::from_record_key(&key); - info!("Existing record loaded: {path:?}"); - return Some((key, (address, record_type))); + } } None }; @@ -348,6 +367,7 @@ impl NodeRecordStore { config: NodeRecordStoreConfig, network_event_sender: mpsc::Sender, swarm_cmd_sender: mpsc::Sender, + #[cfg(feature = "open-metrics")] record_count_metric: Option, ) -> Self { info!("Using encryption_seed of {:?}", config.encryption_seed); let encryption_details = derive_aes256gcm_siv_from_seed(&config.encryption_seed); @@ -369,10 +389,10 @@ impl NodeRecordStore { let local_address = NetworkAddress::from_peer(local_id); // Initialize records_by_distance - let mut records_by_distance: BTreeMap = BTreeMap::new(); - for (key, (addr, _record_type)) in records.iter() { - let distance = convert_distance_to_u256(&local_address.distance(addr)); - let _ = records_by_distance.insert(distance, key.clone()); + let mut records_by_distance: BTreeMap = BTreeMap::new(); + for (key, (addr, _record_type, _data_type)) in records.iter() { + let distance = &local_address.distance(addr); + let _ = records_by_distance.insert(*distance, key.clone()); } let cache_size = config.records_cache_size; @@ -381,12 +401,12 @@ impl NodeRecordStore { config, records, records_by_distance, - records_cache: RecordCache::new(cache_size), + records_cache: RecordCache::new(cache_size, CACHE_TIMEOUT), network_event_sender, local_swarm_cmd_sender: swarm_cmd_sender, responsible_distance_range: None, #[cfg(feature = "open-metrics")] - record_count_metric: None, + record_count_metric, received_payment_count, encryption_details, timestamp, @@ -397,18 +417,16 @@ impl NodeRecordStore { record_store.flush_historic_quoting_metrics(); - record_store - } + #[cfg(feature = "open-metrics")] + if let Some(metric) = &record_store.record_count_metric { + let _ = metric.set(record_store.records.len() as i64); + } - /// Set the record_count_metric to report the number of records stored to the metrics server - #[cfg(feature = "open-metrics")] - pub fn set_record_count_metric(mut self, metric: Gauge) -> Self { - self.record_count_metric = Some(metric); - self + record_store } /// Returns the current distance ilog2 (aka bucket) range of CLOSE_GROUP nodes. - pub fn get_responsible_distance_range(&self) -> Option { + pub fn get_responsible_distance_range(&self) -> Option { self.responsible_distance_range } @@ -434,24 +452,17 @@ impl NodeRecordStore { key: &Key, encryption_details: &(Aes256GcmSiv, [u8; 4]), ) -> Option> { - let mut record = Record { - key: key.clone(), - value: bytes, - publisher: None, - expires: None, - }; - - // if we're not encrypting, lets just return the record - if !cfg!(feature = "encrypt-records") { - return Some(Cow::Owned(record)); - } - let (cipher, nonce_starter) = encryption_details; let nonce = generate_nonce_for_record(nonce_starter, key); - match cipher.decrypt(&nonce, record.value.as_ref()) { + match cipher.decrypt(&nonce, bytes.as_slice()) { Ok(value) => { - record.value = value; + let record = Record { + key: key.clone(), + value, + publisher: None, + expires: None, + }; Some(Cow::Owned(record)) } Err(error) => { @@ -592,32 +603,40 @@ impl NodeRecordStore { /// Returns the set of `NetworkAddress::RecordKey` held by the store /// Use `record_addresses_ref` to get a borrowed type - pub(crate) fn record_addresses(&self) -> HashMap { + pub(crate) fn record_addresses(&self) -> HashMap { self.records .iter() - .map(|(_record_key, (addr, record_type))| (addr.clone(), record_type.clone())) + .map(|(_record_key, (addr, record_type, _data_type))| { + (addr.clone(), record_type.clone()) + }) .collect() } /// Returns the reference to the set of `NetworkAddress::RecordKey` held by the store - pub(crate) fn record_addresses_ref(&self) -> &HashMap { + pub(crate) fn record_addresses_ref( + &self, + ) -> &HashMap { &self.records } /// The follow up to `put_verified`, this only registers the RecordKey /// in the RecordStore records set. After this it should be safe /// to return the record as stored. - pub(crate) fn mark_as_stored(&mut self, key: Key, record_type: RecordType) { + pub(crate) fn mark_as_stored( + &mut self, + key: Key, + validate_type: ValidationType, + data_type: DataTypes, + ) { let addr = NetworkAddress::from_record_key(&key); let distance = self.local_address.distance(&addr); - let distance_u256 = convert_distance_to_u256(&distance); // Update main records store self.records - .insert(key.clone(), (addr.clone(), record_type)); + .insert(key.clone(), (addr.clone(), validate_type, data_type)); // Update bucket index - let _ = self.records_by_distance.insert(distance_u256, key.clone()); + let _ = self.records_by_distance.insert(distance, key.clone()); // Update farthest record if needed (unchanged) if let Some((_farthest_record, farthest_record_distance)) = self.farthest_record.clone() { @@ -630,15 +649,11 @@ impl NodeRecordStore { } /// Prepare record bytes for storage - /// If feats are enabled, this will eg, encrypt the record for storage + /// This will encrypt the record for storage fn prepare_record_bytes( record: Record, encryption_details: (Aes256GcmSiv, [u8; 4]), ) -> Option> { - if !cfg!(feature = "encrypt-records") { - return Some(record.value); - } - let (cipher, nonce_starter) = encryption_details; let nonce = generate_nonce_for_record(&nonce_starter, &record.key); @@ -659,7 +674,12 @@ impl NodeRecordStore { /// /// The record is marked as written to disk once `mark_as_stored` is called, /// this avoids us returning half-written data or registering it as stored before it is. - pub(crate) fn put_verified(&mut self, r: Record, record_type: RecordType) -> Result<()> { + pub(crate) fn put_verified( + &mut self, + r: Record, + record_type: ValidationType, + is_client_put: bool, + ) -> Result<()> { let key = &r.key; let record_key = PrettyPrintRecordKey::from(&r.key).into_owned(); debug!("PUTting a verified Record: {record_key:?}"); @@ -678,8 +698,10 @@ impl NodeRecordStore { } } - // Store the new record to the cache - self.records_cache.push_back(key.clone(), r.clone()); + // Only cash the record that put by client. For a quick response to the ChunkProof check. + if is_client_put { + self.records_cache.push_back(key.clone(), r.clone()); + } self.prune_records_if_needed(key)?; @@ -697,13 +719,26 @@ impl NodeRecordStore { let record_key2 = record_key.clone(); spawn(async move { let key = r.key.clone(); + let data_type = match RecordHeader::get_data_type(&r) { + Ok(data_type) => data_type, + Err(err) => { + error!( + "Error get data_type of record {record_key2:?} filename: {filename}, error: {err:?}" + ); + return; + } + }; if let Some(bytes) = Self::prepare_record_bytes(r, encryption_details) { let cmd = match fs::write(&file_path, bytes) { Ok(_) => { // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): info!("Wrote record {record_key2:?} to disk! filename: {filename}"); - LocalSwarmCmd::AddLocalRecordAsStored { key, record_type } + LocalSwarmCmd::AddLocalRecordAsStored { + key, + record_type, + data_type, + } } Err(err) => { error!( @@ -725,9 +760,12 @@ impl NodeRecordStore { pub(crate) fn quoting_metrics( &self, key: &Key, + data_type: u32, + data_size: usize, network_size: Option, ) -> (QuotingMetrics, bool) { let records_stored = self.records.len(); + let records_per_type = self.records_per_type(); let live_time = if let Ok(elapsed) = self.timestamp.elapsed() { elapsed.as_secs() @@ -736,7 +774,10 @@ impl NodeRecordStore { }; let mut quoting_metrics = QuotingMetrics { + data_type, + data_size, close_records_stored: records_stored, + records_per_type, max_records: self.config.max_records, received_payment_count: self.received_payment_count, live_time, @@ -748,7 +789,7 @@ impl NodeRecordStore { let relevant_records = self.get_records_within_distance_range(distance_range); // The `responsible_range` is the network density - quoting_metrics.network_density = Some(distance_range.to_be_bytes()); + quoting_metrics.network_density = Some(distance_range.0.to_big_endian()); quoting_metrics.close_records_stored = relevant_records; } else { @@ -771,7 +812,7 @@ impl NodeRecordStore { } /// Calculate how many records are stored within a distance range - pub fn get_records_within_distance_range(&self, range: U256) -> usize { + pub fn get_records_within_distance_range(&self, range: Distance) -> usize { let within_range = self .records_by_distance .range(..range) @@ -784,9 +825,17 @@ impl NodeRecordStore { } /// Setup the distance range. - pub(crate) fn set_responsible_distance_range(&mut self, responsible_distance: U256) { + pub(crate) fn set_responsible_distance_range(&mut self, responsible_distance: Distance) { self.responsible_distance_range = Some(responsible_distance); } + + fn records_per_type(&self) -> Vec<(u32, u32)> { + let mut map = BTreeMap::new(); + for (_, _, data_type) in self.records.values() { + *map.entry(data_type.get_index()).or_insert(0) += 1; + } + map.into_iter().collect() + } } impl RecordStore for NodeRecordStore { @@ -831,20 +880,25 @@ impl RecordStore for NodeRecordStore { match RecordHeader::from_record(&record) { Ok(record_header) => { match record_header.kind { - RecordKind::ChunkWithPayment | RecordKind::RegisterWithPayment => { + RecordKind::DataWithPayment(_) => { debug!("Record {record_key:?} with payment shall always be processed."); } - _ => { + // Shall not use wildcard, to avoid mis-match during enum update. + RecordKind::DataOnly(_) => { // Chunk with existing key do not to be stored again. - // `Spend` or `Register` with same content_hash do not to be stored again, - // otherwise shall be passed further to allow - // double transaction to be detected or register op update. + // Others with same content_hash do not to be stored again, + // otherwise shall be passed further to allow different version of nonchunk + // to be detected or updated. match self.records.get(&record.key) { - Some((_addr, RecordType::Chunk)) => { + Some((_addr, ValidationType::Chunk, _data_type)) => { debug!("Chunk {record_key:?} already exists."); return Ok(()); } - Some((_addr, RecordType::NonChunk(existing_content_hash))) => { + Some(( + _addr, + ValidationType::NonChunk(existing_content_hash), + _data_type, + )) => { let content_hash = XorName::from_content(&record.value); if content_hash == *existing_content_hash { debug!("A non-chunk record {record_key:?} with same content_hash {content_hash:?} already exists."); @@ -879,8 +933,8 @@ impl RecordStore for NodeRecordStore { fn remove(&mut self, k: &Key) { // Remove from main store - if let Some((addr, _)) = self.records.remove(k) { - let distance = convert_distance_to_u256(&self.local_address.distance(&addr)); + if let Some((addr, _, _)) = self.records.remove(k) { + let distance = self.local_address.distance(&addr); let _ = self.records_by_distance.remove(&distance); } @@ -939,29 +993,7 @@ impl RecordStore for NodeRecordStore { /// A place holder RecordStore impl for the client that does nothing #[derive(Default, Debug)] -pub struct ClientRecordStore { - empty_record_addresses: HashMap, -} - -impl ClientRecordStore { - pub(crate) fn contains(&self, _key: &Key) -> bool { - false - } - - pub(crate) fn record_addresses(&self) -> HashMap { - HashMap::new() - } - - pub(crate) fn record_addresses_ref(&self) -> &HashMap { - &self.empty_record_addresses - } - - pub(crate) fn put_verified(&mut self, _r: Record, _record_type: RecordType) -> Result<()> { - Ok(()) - } - - pub(crate) fn mark_as_stored(&mut self, _r: Key, _t: RecordType) {} -} +pub struct ClientRecordStore {} impl RecordStore for ClientRecordStore { type RecordsIter<'a> = vec::IntoIter>; @@ -1003,9 +1035,8 @@ mod tests { use bls::SecretKey; use xor_name::XorName; - use ant_protocol::convert_distance_to_u256; use ant_protocol::storage::{ - try_deserialize_record, try_serialize_record, Chunk, ChunkAddress, Scratchpad, + try_deserialize_record, try_serialize_record, Chunk, ChunkAddress, DataTypes, Scratchpad, }; use assert_fs::{ fixture::{PathChild, PathCreateDir}, @@ -1038,7 +1069,7 @@ mod tests { fn arbitrary(g: &mut Gen) -> ArbitraryRecord { let value = match try_serialize_record( &(0..50).map(|_| rand::random::()).collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), @@ -1076,6 +1107,8 @@ mod tests { Default::default(), network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); // An initial unverified put should not write to disk @@ -1095,12 +1128,12 @@ mod tests { let returned_record_key = returned_record.key.clone(); assert!(store - .put_verified(returned_record, RecordType::Chunk) + .put_verified(returned_record, ValidationType::Chunk, true) .is_ok()); // We must also mark the record as stored (which would be triggered after the async write in nodes // via NetworkEvent::CompletedWrite) - store.mark_as_stored(returned_record_key, RecordType::Chunk); + store.mark_as_stored(returned_record_key, ValidationType::Chunk, DataTypes::Chunk); // loop over store.get max_iterations times to ensure async disk write had time to complete. let max_iterations = 10; @@ -1144,14 +1177,18 @@ mod tests { ..Default::default() }; let self_id = PeerId::random(); - let (network_event_sender, _) = mpsc::channel(1); - let (swarm_cmd_sender, _) = mpsc::channel(1); + + // Create channels with proper receivers + let (network_event_sender, _network_event_receiver) = mpsc::channel(1); + let (swarm_cmd_sender, mut swarm_cmd_receiver) = mpsc::channel(1); let mut store = NodeRecordStore::with_config( self_id, store_config.clone(), network_event_sender.clone(), swarm_cmd_sender.clone(), + #[cfg(feature = "open-metrics")] + None, ); // Create a chunk @@ -1162,41 +1199,62 @@ mod tests { // Create a record from the chunk let record = Record { key: NetworkAddress::ChunkAddress(chunk_address).to_record_key(), - value: try_serialize_record(&chunk, RecordKind::Chunk)?.to_vec(), + value: try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk))?.to_vec(), expires: None, publisher: None, }; // Store the chunk using put_verified assert!(store - .put_verified(record.clone(), RecordType::Chunk) + .put_verified(record.clone(), ValidationType::Chunk, true) .is_ok()); - // Mark as stored (simulating the CompletedWrite event) - store.mark_as_stored(record.key.clone(), RecordType::Chunk); + // Wait for the async write operation to complete + if let Some(cmd) = swarm_cmd_receiver.recv().await { + match cmd { + LocalSwarmCmd::AddLocalRecordAsStored { + key, + record_type, + data_type, + } => { + store.mark_as_stored(key, record_type, data_type); + } + _ => panic!("Unexpected command received"), + } + } // Verify the chunk is stored let stored_record = store.get(&record.key); - assert!(stored_record.is_some(), "Chunk should be stored"); + assert!(stored_record.is_some(), "Chunk should be stored initially"); // Sleep a while to let OS completes the flush to disk - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(1)).await; + + // Create new channels for the restarted store + let (new_network_event_sender, _new_network_event_receiver) = mpsc::channel(1); + let (new_swarm_cmd_sender, _new_swarm_cmd_receiver) = mpsc::channel(1); - // Restart the store with same encrypt_seed + // Restart the store with same encrypt_seed but new channels drop(store); let store = NodeRecordStore::with_config( self_id, store_config, - network_event_sender.clone(), - swarm_cmd_sender.clone(), + new_network_event_sender, + new_swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); - // Sleep a lit bit to let OS completes restoring - sleep(Duration::from_secs(1)).await; - // Verify the record still exists let stored_record = store.get(&record.key); - assert!(stored_record.is_some(), "Chunk should be stored"); + assert!( + stored_record.is_some(), + "Chunk should be stored after restart with same key" + ); + + // Create new channels for the different seed test + let (diff_network_event_sender, _diff_network_event_receiver) = mpsc::channel(1); + let (diff_swarm_cmd_sender, _diff_swarm_cmd_receiver) = mpsc::channel(1); // Restart the store with different encrypt_seed let self_id_diff = PeerId::random(); @@ -1208,25 +1266,18 @@ mod tests { let store_diff = NodeRecordStore::with_config( self_id_diff, store_config_diff, - network_event_sender, - swarm_cmd_sender, + diff_network_event_sender, + diff_swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); - // Sleep a lit bit to let OS completes restoring (if has) - sleep(Duration::from_secs(1)).await; - - // Verify the record existence, shall get removed when encryption enabled - if cfg!(feature = "encrypt-records") { - assert!( - store_diff.get(&record.key).is_none(), - "Chunk should be gone" - ); - } else { - assert!( - store_diff.get(&record.key).is_some(), - "Chunk shall persists without encryption" - ); - } + // When encryption is enabled, the record should be gone because it can't be decrypted + // with the different encryption seed + assert!( + store_diff.get(&record.key).is_none(), + "Chunk should be gone with different encryption key" + ); Ok(()) } @@ -1247,6 +1298,8 @@ mod tests { store_config, network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); // Create a chunk @@ -1264,11 +1317,11 @@ mod tests { // Store the chunk using put_verified assert!(store - .put_verified(record.clone(), RecordType::Chunk) + .put_verified(record.clone(), ValidationType::Chunk, true) .is_ok()); // Mark as stored (simulating the CompletedWrite event) - store.mark_as_stored(record.key.clone(), RecordType::Chunk); + store.mark_as_stored(record.key.clone(), ValidationType::Chunk, DataTypes::Chunk); // Verify the chunk is stored let stored_record = store.get(&record.key); @@ -1311,24 +1364,23 @@ mod tests { store_config, network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); // Create a scratchpad let unencrypted_scratchpad_data = Bytes::from_static(b"Test scratchpad data"); let owner_sk = SecretKey::random(); - let owner_pk = owner_sk.public_key(); - - let mut scratchpad = Scratchpad::new(owner_pk, 0); - let _next_version = - scratchpad.update_and_sign(unencrypted_scratchpad_data.clone(), &owner_sk); + let scratchpad = Scratchpad::new(&owner_sk, 0, &unencrypted_scratchpad_data, 0); let scratchpad_address = *scratchpad.address(); // Create a record from the scratchpad let record = Record { key: NetworkAddress::ScratchpadAddress(scratchpad_address).to_record_key(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad)?.to_vec(), + value: try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad))? + .to_vec(), expires: None, publisher: None, }; @@ -1337,14 +1389,16 @@ mod tests { assert!(store .put_verified( record.clone(), - RecordType::NonChunk(XorName::from_content(&record.value)) + ValidationType::NonChunk(XorName::from_content(&record.value)), + true, ) .is_ok()); // Mark as stored (simulating the CompletedWrite event) store.mark_as_stored( record.key.clone(), - RecordType::NonChunk(XorName::from_content(&record.value)), + ValidationType::NonChunk(XorName::from_content(&record.value)), + DataTypes::Scratchpad, ); // Verify the scratchpad is stored @@ -1404,6 +1458,8 @@ mod tests { store_config.clone(), network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); // keep track of everything ever stored, to check missing at the end are further away let mut stored_records_at_some_point: Vec = vec![]; @@ -1418,7 +1474,7 @@ mod tests { let record_key = NetworkAddress::from_peer(PeerId::random()).to_record_key(); let value = match try_serialize_record( &(0..50).map(|_| rand::random::()).collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), @@ -1431,7 +1487,9 @@ mod tests { }; // Will be stored anyway. - let succeeded = store.put_verified(record, RecordType::Chunk).is_ok(); + let succeeded = store + .put_verified(record, ValidationType::Chunk, true) + .is_ok(); if !succeeded { failed_records.push(record_key.clone()); @@ -1439,7 +1497,7 @@ mod tests { } else { // We must also mark the record as stored (which would be triggered // after the async write in nodes via NetworkEvent::CompletedWrite) - store.mark_as_stored(record_key.clone(), RecordType::Chunk); + store.mark_as_stored(record_key.clone(), ValidationType::Chunk, DataTypes::Chunk); println!("success sotred len: {:?} ", store.record_addresses().len()); stored_records_at_some_point.push(record_key.clone()); @@ -1493,7 +1551,7 @@ mod tests { // now for any stored data. It either shoudl still be stored OR further away than `most_distant_data` for data in stored_records_at_some_point { let data_addr = NetworkAddress::from_record_key(&data); - if !sorted_stored_data.contains(&(&data_addr, &RecordType::Chunk)) { + if !sorted_stored_data.contains(&(&data_addr, &ValidationType::Chunk)) { assert!( self_address.distance(&data_addr) > self_address.distance(most_distant_data), @@ -1528,6 +1586,8 @@ mod tests { store_config, network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); let mut stored_records: Vec = vec![]; @@ -1541,7 +1601,7 @@ mod tests { &(0..max_records) .map(|_| rand::random::()) .collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), @@ -1552,12 +1612,14 @@ mod tests { publisher: None, expires: None, }; - assert!(store.put_verified(record, RecordType::Chunk).is_ok()); + assert!(store + .put_verified(record, ValidationType::Chunk, true) + .is_ok()); // We must also mark the record as stored (which would be triggered after the async write in nodes // via NetworkEvent::CompletedWrite) - store.mark_as_stored(record_key.clone(), RecordType::Chunk); + store.mark_as_stored(record_key.clone(), ValidationType::Chunk, DataTypes::Chunk); - stored_records.push(record_key); + stored_records.push(record_key.clone()); stored_records.sort_by(|a, b| { let a = NetworkAddress::from_record_key(a); let b = NetworkAddress::from_record_key(b); @@ -1572,12 +1634,12 @@ mod tests { .wrap_err("Could not parse record store key")?, ); // get the distance to this record from our local key - let distance = convert_distance_to_u256(&self_address.distance(&halfway_record_address)); + let distance = &self_address.distance(&halfway_record_address); // must be plus one bucket from the halfway record - store.set_responsible_distance_range(distance); + store.set_responsible_distance_range(*distance); - let records_in_range = store.get_records_within_distance_range(distance); + let records_in_range = store.get_records_within_distance_range(*distance); // check that the number of records returned is larger than half our records // (ie, that we cover _at least_ all the records within our distance range) @@ -1612,6 +1674,8 @@ mod tests { store_config.clone(), network_event_sender.clone(), swarm_cmd_sender.clone(), + #[cfg(feature = "open-metrics")] + None, ); store.payment_received(); @@ -1624,6 +1688,8 @@ mod tests { store_config, network_event_sender, swarm_cmd_sender, + #[cfg(feature = "open-metrics")] + None, ); assert_eq!(1, new_store.received_payment_count); @@ -1631,4 +1697,77 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_cache_pruning_and_size_limit() { + // Create cache with small size and short timeout for testing + let cache_size = 3; + let cache_timeout = Duration::from_millis(100); + let mut cache = RecordCache::new(cache_size, cache_timeout); + + // Create test records + let record1 = Record { + key: RecordKey::new(b"key1"), + value: b"value1".to_vec(), + publisher: None, + expires: None, + }; + let record2 = Record { + key: RecordKey::new(b"key2"), + value: b"value2".to_vec(), + publisher: None, + expires: None, + }; + let record3 = Record { + key: RecordKey::new(b"key3"), + value: b"value3".to_vec(), + publisher: None, + expires: None, + }; + let record4 = Record { + key: RecordKey::new(b"key4"), + value: b"value4".to_vec(), + publisher: None, + expires: None, + }; + + // Add records up to cache size + cache.push_back(record1.key.clone(), record1.clone()); + cache.push_back(record2.key.clone(), record2.clone()); + cache.push_back(record3.key.clone(), record3.clone()); + + // Verify all records are present + assert!(cache.get(&record1.key).is_some()); + assert!(cache.get(&record2.key).is_some()); + assert!(cache.get(&record3.key).is_some()); + + // Add one more record to trigger size-based pruning + cache.push_back(record4.key.clone(), record4.clone()); + + // Verify cache size is maintained + assert_eq!(cache.records_cache.len(), cache_size); + + // Verify oldest record was removed + assert!(cache.get(&record1.key).is_none()); + + // Wait for timeout to expire + sleep(cache_timeout + Duration::from_millis(10)).await; + + // Add another record to trigger time-based pruning + let record5 = Record { + key: RecordKey::new(b"key5"), + value: b"value5".to_vec(), + publisher: None, + expires: None, + }; + cache.push_back(record5.key.clone(), record5.clone()); + + // Verify all timed-out records were removed + assert!(cache.get(&record2.key).is_none()); + assert!(cache.get(&record3.key).is_none()); + assert!(cache.get(&record4.key).is_none()); + + // Verify new record is present + assert!(cache.get(&record5.key).is_some()); + } } diff --git a/ant-networking/src/record_store_api.rs b/ant-networking/src/record_store_api.rs index 0955d5499f..e13e1fdd44 100644 --- a/ant-networking/src/record_store_api.rs +++ b/ant-networking/src/record_store_api.rs @@ -7,12 +7,15 @@ // permissions and limitations relating to use of the SAFE Network Software. #![allow(clippy::mutable_key_type)] // for the Bytes in NetworkAddress +use crate::error::{NetworkError, Result}; use crate::record_store::{ClientRecordStore, NodeRecordStore}; -use ant_evm::{QuotingMetrics, U256}; -use ant_protocol::{storage::RecordType, NetworkAddress}; +use ant_evm::QuotingMetrics; +use ant_protocol::{ + storage::{DataTypes, ValidationType}, + NetworkAddress, +}; use libp2p::kad::{ - store::{RecordStore, Result}, - ProviderRecord, Record, RecordKey, + store::RecordStore, KBucketDistance as Distance, ProviderRecord, Record, RecordKey, }; use std::{borrow::Cow, collections::HashMap}; @@ -20,7 +23,6 @@ pub enum UnifiedRecordStore { Client(ClientRecordStore), Node(NodeRecordStore), } - impl RecordStore for UnifiedRecordStore { type RecordsIter<'a> = std::vec::IntoIter>; type ProvidedIter<'a> = std::vec::IntoIter>; @@ -32,7 +34,7 @@ impl RecordStore for UnifiedRecordStore { } } - fn put(&mut self, r: Record) -> Result<()> { + fn put(&mut self, r: Record) -> libp2p::kad::store::Result<()> { match self { Self::Client(store) => store.put(r), Self::Node(store) => store.put(r), @@ -53,7 +55,7 @@ impl RecordStore for UnifiedRecordStore { } } - fn add_provider(&mut self, record: ProviderRecord) -> Result<()> { + fn add_provider(&mut self, record: ProviderRecord) -> libp2p::kad::store::Result<()> { match self { Self::Client(store) => store.add_provider(record), Self::Node(store) => store.add_provider(record), @@ -83,31 +85,50 @@ impl RecordStore for UnifiedRecordStore { } impl UnifiedRecordStore { - pub(crate) fn contains(&self, key: &RecordKey) -> bool { + pub(crate) fn contains(&self, key: &RecordKey) -> Result { match self { - Self::Client(store) => store.contains(key), - Self::Node(store) => store.contains(key), + Self::Client(_) => { + error!("Calling 'contains' at Client. This should not happen"); + Err(NetworkError::OperationNotAllowedOnClientRecordStore) + } + Self::Node(store) => Ok(store.contains(key)), } } - pub(crate) fn record_addresses(&self) -> HashMap { + pub(crate) fn record_addresses(&self) -> Result> { match self { - Self::Client(store) => store.record_addresses(), - Self::Node(store) => store.record_addresses(), + Self::Client(_) => { + error!("Calling record_addresses at Client. This should not happen"); + Err(NetworkError::OperationNotAllowedOnClientRecordStore) + } + Self::Node(store) => Ok(store.record_addresses()), } } - pub(crate) fn record_addresses_ref(&self) -> &HashMap { + pub(crate) fn record_addresses_ref( + &self, + ) -> Result<&HashMap> { match self { - Self::Client(store) => store.record_addresses_ref(), - Self::Node(store) => store.record_addresses_ref(), + Self::Client(_) => { + error!("Calling record_addresses_ref at Client. This should not happen"); + Err(NetworkError::OperationNotAllowedOnClientRecordStore) + } + Self::Node(store) => Ok(store.record_addresses_ref()), } } - pub(crate) fn put_verified(&mut self, r: Record, record_type: RecordType) -> Result<()> { + pub(crate) fn put_verified( + &mut self, + r: Record, + record_type: ValidationType, + is_client_put: bool, + ) -> libp2p::kad::store::Result<()> { match self { - Self::Client(store) => store.put_verified(r, record_type), - Self::Node(store) => store.put_verified(r, record_type), + Self::Client(_) => { + error!("Calling put_verified at Client. This should not happen"); + Ok(()) + } + Self::Node(store) => store.put_verified(r, record_type, is_client_put), } } @@ -116,69 +137,77 @@ impl UnifiedRecordStore { pub(crate) fn quoting_metrics( &self, key: &RecordKey, + data_type: u32, + data_size: usize, network_size: Option, - ) -> (QuotingMetrics, bool) { + ) -> Result<(QuotingMetrics, bool)> { match self { Self::Client(_) => { - warn!("Calling quoting metrics calculation at Client. This should not happen"); - Default::default() + error!("Calling quoting_metrics at Client. This should not happen"); + Err(NetworkError::OperationNotAllowedOnClientRecordStore) } - Self::Node(store) => store.quoting_metrics(key, network_size), + Self::Node(store) => Ok(store.quoting_metrics(key, data_type, data_size, network_size)), } } pub(crate) fn payment_received(&mut self) { match self { Self::Client(_) => { - warn!("Calling payment_received at Client. This should not happen"); + error!("Calling payment_received at Client. This should not happen"); } Self::Node(store) => store.payment_received(), } } - pub(crate) fn get_farthest_replication_distance(&self) -> Option { + pub(crate) fn get_farthest_replication_distance(&self) -> Result> { match self { - Self::Client(_store) => { - warn!("Calling get_distance_range at Client. This should not happen"); - None + Self::Client(_) => { + error!( + "Calling get_farthest_replication_distance at Client. This should not happen" + ); + Err(NetworkError::OperationNotAllowedOnClientRecordStore) } - Self::Node(store) => store.get_responsible_distance_range(), + Self::Node(store) => Ok(store.get_responsible_distance_range()), } } - pub(crate) fn set_distance_range(&mut self, distance: U256) { + pub(crate) fn set_distance_range(&mut self, distance: Distance) { match self { - Self::Client(_store) => { - warn!("Calling set_distance_range at Client. This should not happen"); + Self::Client(_) => { + error!("Calling set_distance_range at Client. This should not happen"); } Self::Node(store) => store.set_responsible_distance_range(distance), } } - pub(crate) fn get_farthest(&self) -> Option { + pub(crate) fn get_farthest(&self) -> Result> { match self { - Self::Client(_store) => { - warn!("Calling get_farthest at Client. This should not happen"); - None - } - Self::Node(store) => store.get_farthest(), + Self::Client(_) => Err(NetworkError::OperationNotAllowedOnClientRecordStore), + Self::Node(store) => Ok(store.get_farthest()), } } /// Mark the record as stored in the store. /// This adds it to records set, so it can now be retrieved /// (to be done after writes are finalised) - pub(crate) fn mark_as_stored(&mut self, k: RecordKey, record_type: RecordType) { + pub(crate) fn mark_as_stored( + &mut self, + k: RecordKey, + record_type: ValidationType, + data_type: DataTypes, + ) { match self { - Self::Client(store) => store.mark_as_stored(k, record_type), - Self::Node(store) => store.mark_as_stored(k, record_type), + Self::Client(_) => { + error!("Calling mark_as_stored at Client. This should not happen"); + } + Self::Node(store) => store.mark_as_stored(k, record_type, data_type), }; } pub(crate) fn cleanup_irrelevant_records(&mut self) { match self { Self::Client(_store) => { - warn!("Calling cleanup_irrelevant_records at Client. This should not happen"); + error!("Calling cleanup_irrelevant_records at Client. This should not happen"); } Self::Node(store) => store.cleanup_irrelevant_records(), } diff --git a/ant-networking/src/relay_manager.rs b/ant-networking/src/relay_manager.rs index 92a1fb8888..896467a4f3 100644 --- a/ant-networking/src/relay_manager.rs +++ b/ant-networking/src/relay_manager.rs @@ -8,48 +8,128 @@ use crate::driver::{BadNodes, NodeBehaviour}; use itertools::Itertools; +use libp2p::swarm::ConnectionId; use libp2p::{ core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm, }; +#[cfg(feature = "open-metrics")] +use prometheus_client::metrics::gauge::Gauge; use rand::Rng; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +#[cfg(feature = "open-metrics")] +use std::sync::atomic::AtomicU64; +use std::time::Instant; +#[cfg(feature = "open-metrics")] +use std::{collections::btree_map::Entry, time::SystemTime}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 4; const MAX_POTENTIAL_CANDIDATES: usize = 1000; +/// We could get multiple incoming connections from the same peer through multiple relay servers, and only one of them +/// would succeed. So we wait and collect all such connections 'from' peer, instead of just recording the +/// success/failure for each connection. +#[cfg(feature = "open-metrics")] +const MAX_DURATION_TO_TRACK_INCOMING_CONNECTIONS_PER_PEER: std::time::Duration = + std::time::Duration::from_secs(20); + +#[cfg(feature = "open-metrics")] +const RESERVATION_SCORE_ROLLING_WINDOW: usize = 100; + +/// The connections from a single peer through multiple relay servers. +/// It is a vector of (relay server, connection id, time of connection, Option). +/// A None value for success/failure means that the connection is still pending. +#[cfg(feature = "open-metrics")] +type ConnectionsFromPeer = Vec<(PeerId, ConnectionId, SystemTime, Option)>; + pub(crate) fn is_a_relayed_peer(addrs: &HashSet) -> bool { addrs .iter() .any(|multiaddr| multiaddr.iter().any(|p| matches!(p, Protocol::P2pCircuit))) } -/// Manage the relay servers that we are connected to. +/// Manage the relay servers that a private node is connected to. /// This is the client side of the relay server protocol. #[derive(Debug)] pub(crate) struct RelayManager { self_peer_id: PeerId, - candidates: VecDeque<(PeerId, Multiaddr)>, + /// The potential relay servers that we can connect to. + relay_server_candidates: VecDeque<(PeerId, Multiaddr)>, + /// The relay servers that we are waiting for a reservation from. waiting_for_reservation: BTreeMap, - connected_relays: BTreeMap, - + /// The relay servers that we are connected to. + connected_relay_servers: BTreeMap, /// Tracker for the relayed listen addresses. relayed_listener_id_map: HashMap, + #[cfg(feature = "open-metrics")] + /// Health of the relayed connections. + reservation_health: Option, +} + +#[cfg(feature = "open-metrics")] +#[derive(Debug)] +struct RelayReservationHealth { + /// We could have multiple incoming connections from the same peer through multiple relay servers. But we could + /// just have a single one as well. + /// This is a map of the 'from peer' to the multiple relay servers that the incoming connections are coming through. + incoming_connections_from_remote_peer: BTreeMap, + /// A rolling window of reservation score per relay server. + reservation_score: BTreeMap, + /// To track the avg health of all the reservations. + relay_reservation_health_metric: Gauge, +} + +#[cfg(feature = "open-metrics")] +#[derive(Debug, Default, Clone)] +struct ReservationStat { + stat: VecDeque, +} + +#[cfg(feature = "open-metrics")] +impl ReservationStat { + fn record_value(&mut self, value: bool) { + self.stat.push_back(value); + if self.stat.len() > RESERVATION_SCORE_ROLLING_WINDOW { + self.stat.pop_front(); + } + } + + fn success_rate(&self) -> f64 { + let success = self.stat.iter().filter(|s| **s).count(); + let error = self.stat.len() - success; + + if success + error == 0 { + 0.0 + } else { + success as f64 / (success + error) as f64 + } + } } impl RelayManager { pub(crate) fn new(self_peer_id: PeerId) -> Self { Self { self_peer_id, - connected_relays: Default::default(), + connected_relay_servers: Default::default(), waiting_for_reservation: Default::default(), - candidates: Default::default(), + relay_server_candidates: Default::default(), relayed_listener_id_map: Default::default(), + #[cfg(feature = "open-metrics")] + reservation_health: None, } } + #[cfg(feature = "open-metrics")] + pub(crate) fn set_reservation_health_metrics(&mut self, gauge: Gauge) { + self.reservation_health = Some(RelayReservationHealth { + incoming_connections_from_remote_peer: Default::default(), + reservation_score: Default::default(), + relay_reservation_health_metric: gauge, + }) + } + /// Should we keep this peer alive? Closing a connection to that peer would remove that server from the listen addr. pub(crate) fn keep_alive_peer(&self, peer_id: &PeerId) -> bool { - self.connected_relays.contains_key(peer_id) + self.connected_relay_servers.contains_key(peer_id) || self.waiting_for_reservation.contains_key(peer_id) } @@ -61,7 +141,7 @@ impl RelayManager { addrs: &HashSet, stream_protocols: &Vec, ) { - if self.candidates.len() >= MAX_POTENTIAL_CANDIDATES { + if self.relay_server_candidates.len() >= MAX_POTENTIAL_CANDIDATES { return; } @@ -72,7 +152,8 @@ impl RelayManager { // Hence here can add the addr directly. if let Some(relay_addr) = Self::craft_relay_address(addr, Some(*peer_id)) { debug!("Adding {peer_id:?} with {relay_addr:?} as a potential relay candidate"); - self.candidates.push_back((*peer_id, relay_addr)); + self.relay_server_candidates + .push_back((*peer_id, relay_addr)); } } } else { @@ -88,13 +169,14 @@ impl RelayManager { swarm: &mut Swarm, bad_nodes: &BadNodes, ) { - if self.connected_relays.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS - || self.candidates.is_empty() + if self.connected_relay_servers.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS + || self.relay_server_candidates.is_empty() { return; } - let reservations_to_make = MAX_CONCURRENT_RELAY_CONNECTIONS - self.connected_relays.len(); + let reservations_to_make = + MAX_CONCURRENT_RELAY_CONNECTIONS - self.connected_relay_servers.len(); let mut n_reservations = 0; while n_reservations < reservations_to_make { @@ -102,14 +184,14 @@ impl RelayManager { // we're behind nat? // Pick a random candidate from the vector. Check if empty, or `gen_range` panics for empty range. - let index = if self.candidates.is_empty() { + let index = if self.relay_server_candidates.is_empty() { debug!("No more relay candidates."); break; } else { - rand::thread_rng().gen_range(0..self.candidates.len()) + rand::thread_rng().gen_range(0..self.relay_server_candidates.len()) }; - if let Some((peer_id, relay_addr)) = self.candidates.remove(index) { + if let Some((peer_id, relay_addr)) = self.relay_server_candidates.remove(index) { // skip if detected as a bad node if let Some((_, is_bad)) = bad_nodes.get(&peer_id) { if *is_bad { @@ -118,7 +200,7 @@ impl RelayManager { } } - if self.connected_relays.contains_key(&peer_id) + if self.connected_relay_servers.contains_key(&peer_id) || self.waiting_for_reservation.contains_key(&peer_id) { debug!("We are already using {peer_id:?} as a relay server. Skipping."); @@ -148,24 +230,30 @@ impl RelayManager { &mut self, peer_id: &PeerId, swarm: &mut Swarm, + live_connected_peers: &BTreeMap, ) { - if tracing::level_enabled!(tracing::Level::DEBUG) { - let all_external_addresses = swarm.external_addresses().collect_vec(); - let all_listeners = swarm.listeners().collect_vec(); - debug!("All our listeners: {all_listeners:?}"); - debug!("All our external addresses: {all_external_addresses:?}"); - } - match self.waiting_for_reservation.remove(peer_id) { Some(addr) => { info!("Successfully made reservation with {peer_id:?} on {addr:?}. Adding the addr to external address."); swarm.add_external_address(addr.clone()); - self.connected_relays.insert(*peer_id, addr); + self.connected_relay_servers.insert(*peer_id, addr); } None => { debug!("Made a reservation with a peer that we had not requested to"); } } + + if self.connected_relay_servers.len() == MAX_CONCURRENT_RELAY_CONNECTIONS { + debug!("We have reached the maximum number of relay connections. Push new identify info to all connected peers"); + // send identify to all connected peers. + + swarm.behaviour_mut().identify.push( + live_connected_peers + .values() + .map(|(peer_id, ..)| *peer_id) + .unique(), + ); + } } /// Update client state if the reservation has been cancelled or if the relay has closed. @@ -178,7 +266,7 @@ impl RelayManager { return; }; - if let Some(addr) = self.connected_relays.remove(&peer_id) { + if let Some(addr) = self.connected_relay_servers.remove(&peer_id) { info!("Removing connected relay server as the listener has been closed: {peer_id:?}"); info!("Removing external addr: {addr:?}"); swarm.remove_external_address(&addr); @@ -201,6 +289,43 @@ impl RelayManager { } } + /// Track the incoming connections to monitor the health of a reservation. + #[cfg(feature = "open-metrics")] + pub(crate) fn on_incoming_connection( + &mut self, + connection_id: &ConnectionId, + local_addr: &Multiaddr, + send_back_addr: &Multiaddr, + ) { + if let Some(reservation_health) = &mut self.reservation_health { + reservation_health.on_incoming_connection(connection_id, local_addr, send_back_addr); + } + } + + /// Track the connection established to monitor the health of a reservation. + #[cfg(feature = "open-metrics")] + pub(crate) fn on_connection_established( + &mut self, + from_peer: &PeerId, + connection_id: &ConnectionId, + ) { + if let Some(reservation_health) = &mut self.reservation_health { + reservation_health.on_connection_established(from_peer, connection_id); + } + } + + /// Track the connection error to monitor the health of a reservation. + #[cfg(feature = "open-metrics")] + pub(crate) fn on_incomming_connection_error( + &mut self, + send_back_addr: &Multiaddr, + connection_id: &ConnectionId, + ) { + if let Some(reservation_health) = &mut self.reservation_health { + reservation_health.on_incomming_connection_error(send_back_addr, connection_id); + } + } + fn does_it_support_relay_server_protocol(protocols: &Vec) -> bool { for stream_protocol in protocols { if *stream_protocol == "/libp2p/circuit/relay/0.2.0/stop" { @@ -239,3 +364,204 @@ impl RelayManager { Some(output_addr) } } + +#[cfg(feature = "open-metrics")] +impl RelayReservationHealth { + fn on_incoming_connection( + &mut self, + connection_id: &ConnectionId, + // The local addr would look something like this + // /ip4//udp/39821/quic-v1/p2p/12D3KooWHHVo7euYruLYEZHiwZcHG6p99XqHzjyt8MaZPiEKk5Sp/p2p-circuit + local_addr: &Multiaddr, + // The send back addr would not contain the ip addr, but just the peer ids for private nodes. + // send_back_addr: /p2p/12D3KooWGsKUTLCp6Vi8e9hxUMxAtU5CjPynYKqg77KBco5qBMqD + send_back_addr: &Multiaddr, + ) { + let relay_server = { + if !local_addr + .iter() + .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) + { + debug!("Incoming connection is not routed through a relay server. Not tracking its health."); + return; + }; + + match local_addr.iter().find(|p| matches!(p, Protocol::P2p(_))) { + Some(Protocol::P2p(id)) => id, + _ => { + debug!("Incoming connection does not have a valid 'relay server id'. Not tracking its health."); + return; + } + } + }; + + let from_peer = { + match send_back_addr + .iter() + .find(|p| matches!(p, Protocol::P2p(_))) + { + Some(Protocol::P2p(id)) => id, + _ => { + debug!("Incoming connection does not have a valid 'from peer id'. Not tracking its health."); + return; + } + } + }; + + match self.incoming_connections_from_remote_peer.entry(from_peer) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .push((relay_server, *connection_id, SystemTime::now(), None)); + } + Entry::Vacant(entry) => { + entry.insert(vec![( + relay_server, + *connection_id, + SystemTime::now(), + None, + )]); + } + } + } + + fn on_connection_established(&mut self, from_peer: &PeerId, connection_id: &ConnectionId) { + if let Some(connections) = self + .incoming_connections_from_remote_peer + .get_mut(from_peer) + { + if let Some((_, _, _, succeeded)) = connections + .iter_mut() + .find(|(_, id, _, _)| id == connection_id) + { + *succeeded = Some(true); + } + } + + self.try_update_stat(); + } + + fn on_incomming_connection_error( + &mut self, + send_back_addr: &Multiaddr, + connection_id: &ConnectionId, + ) { + let from_peer = { + match send_back_addr + .iter() + .find(|p| matches!(p, Protocol::P2p(_))) + { + Some(Protocol::P2p(id)) => id, + _ => { + debug!("Incoming connection does not have a valid 'from peer id'. Not tracking its health."); + return; + } + } + }; + + if let Some(connections) = self + .incoming_connections_from_remote_peer + .get_mut(&from_peer) + { + if let Some((_, _, _, succeeded)) = connections + .iter_mut() + .find(|(_, id, _, _)| id == connection_id) + { + *succeeded = Some(false); + } + } + + self.try_update_stat(); + } + + fn try_update_stat(&mut self) { + let mut to_remove = Vec::new(); + + for (from_peer, connections) in self.incoming_connections_from_remote_peer.iter_mut() { + let Some(latest_time) = connections.iter().map(|(_, _, time, _)| time).max() else { + debug!("The incoming connections from {from_peer:?} are empty. Skipping."); + continue; + }; + + let Ok(elapsed) = SystemTime::now().duration_since(*latest_time) else { + debug!("Could not obtain elapsed time."); + continue; + }; + + if elapsed < MAX_DURATION_TO_TRACK_INCOMING_CONNECTIONS_PER_PEER { + continue; + } + + // if at least one connection has been established, we can update the stats. + let mut connection_success = false; + for (relay_server, connection_id, _, _) in connections + .iter() + .filter(|(_, _, _, result)| result.is_some_and(|succeeded| succeeded)) + { + connection_success = true; + debug!("Connection {connection_id:?} from {from_peer:?} through {relay_server:?} has been successful. Increasing the success count"); + match self.reservation_score.entry(*relay_server) { + Entry::Occupied(mut entry) => { + let stat = entry.get_mut(); + stat.record_value(true); + } + Entry::Vacant(entry) => { + let mut stat = ReservationStat::default(); + stat.record_value(true); + entry.insert(stat); + } + } + } + + if !connection_success { + // if none of the connections have been established, we can update the stats. + for (relay_server, connection_id, _, result) in connections.iter() { + if result.is_none() { + debug!("Connection {connection_id:?} from {from_peer:?} through {relay_server:?} is still pending after {elapsed:?}. This is thrown away."); + continue; + }; + debug!("Connection {connection_id:?} from {from_peer:?} through {relay_server:?} is a failure. Increasing the error count"); + match self.reservation_score.entry(*relay_server) { + Entry::Occupied(mut entry) => { + let stat = entry.get_mut(); + stat.record_value(false); + } + Entry::Vacant(entry) => { + let mut stat = ReservationStat::default(); + stat.record_value(false); + entry.insert(stat); + } + } + } + } + + to_remove.push(*from_peer); + } + + for from_peer in to_remove { + self.incoming_connections_from_remote_peer + .remove(&from_peer); + } + + #[cfg(feature = "open-metrics")] + // calculate avg health of all the reservations + let avg_health = self + .reservation_score + .values() + .map(|stat| stat.success_rate()) + .sum::() + / self.reservation_score.len() as f64; + self.relay_reservation_health_metric.set(avg_health); + + self.log_reservation_score(); + } + + fn log_reservation_score(&self) { + for (relay_server, stat) in self.reservation_score.iter() { + debug!( + "Reservation score for {relay_server:?}: {:?}", + stat.success_rate() + ); + } + } +} diff --git a/ant-networking/src/replication_fetcher.rs b/ant-networking/src/replication_fetcher.rs index a009209451..0a65b35684 100644 --- a/ant-networking/src/replication_fetcher.rs +++ b/ant-networking/src/replication_fetcher.rs @@ -7,17 +7,17 @@ // permissions and limitations relating to use of the SAFE Network Software. #![allow(clippy::mutable_key_type)] -use crate::target_arch::spawn; -use crate::{event::NetworkEvent, target_arch::Instant}; -use ant_evm::U256; +use crate::time::spawn; +use crate::{event::NetworkEvent, time::Instant, CLOSE_GROUP_SIZE}; use ant_protocol::{ - convert_distance_to_u256, storage::RecordType, NetworkAddress, PrettyPrintRecordKey, + storage::{DataTypes, ValidationType}, + NetworkAddress, PrettyPrintRecordKey, }; use libp2p::{ kad::{KBucketDistance as Distance, RecordKey, K_VALUE}, PeerId, }; -use std::collections::{hash_map::Entry, BTreeSet, HashMap}; +use std::collections::{hash_map::Entry, BTreeMap, HashMap, HashSet, VecDeque}; use tokio::{sync::mpsc, time::Duration}; // Max parallel fetches that can be undertaken at the same time. @@ -40,16 +40,23 @@ type ReplicationTimeout = Instant; pub(crate) struct ReplicationFetcher { self_peer_id: PeerId, // Pending entries that to be fetched from the target peer. - to_be_fetched: HashMap<(RecordKey, RecordType, PeerId), ReplicationTimeout>, + to_be_fetched: HashMap<(RecordKey, ValidationType, PeerId), ReplicationTimeout>, // Avoid fetching same chunk from different nodes AND carry out too many parallel tasks. - on_going_fetches: HashMap<(RecordKey, RecordType), (PeerId, ReplicationTimeout)>, + on_going_fetches: HashMap<(RecordKey, ValidationType), (PeerId, ReplicationTimeout)>, event_sender: mpsc::Sender, /// Distance range that the incoming key shall be fetched - distance_range: Option, + distance_range: Option, /// Restrict fetch range to closer than this value /// used when the node is full, but we still have "close" data coming in /// that is _not_ closer than our farthest max record farthest_acceptable_distance: Option, + /// Scoring of peers collected from storage_challenge. + /// To be a trustworthy replication source, the peer must has two latest scoring both healthy. + peers_scores: HashMap, Instant)>, + /// During startup, when the knowledge of peers scoring hasn't been built up, + /// only records got `majority` of replicated in copies shall be trusted. + /// This is the temp container to accumulate those intitial replicated in records. + initial_replicates: HashMap<(NetworkAddress, ValidationType), HashSet>, } impl ReplicationFetcher { @@ -62,108 +69,64 @@ impl ReplicationFetcher { event_sender, distance_range: None, farthest_acceptable_distance: None, + peers_scores: HashMap::new(), + initial_replicates: HashMap::new(), } } /// Set the distance range. - pub(crate) fn set_replication_distance_range(&mut self, distance_range: U256) { + pub(crate) fn set_replication_distance_range(&mut self, distance_range: Distance) { self.distance_range = Some(distance_range); } // Adds the non existing incoming keys from the peer to the fetcher. // Returns the next set of keys that has to be fetched from the peer/network. // - // Note: the `incoming_keys` shall already got filter for existence. + // Note: for `fresh_replicate`, the verification is on payment and got undertaken by the caller + // Hence here it shall always be considered as valid to fetch. pub(crate) fn add_keys( &mut self, holder: PeerId, - incoming_keys: Vec<(NetworkAddress, RecordType)>, - locally_stored_keys: &HashMap, + incoming_keys: Vec<(NetworkAddress, ValidationType)>, + locally_stored_keys: &HashMap, + is_fresh_replicate: bool, + closest_k_peers: Vec, ) -> Vec<(PeerId, RecordKey)> { - // Pre-calculate self_address since it's used multiple times - let self_address = NetworkAddress::from_peer(self.self_peer_id); - let total_incoming_keys = incoming_keys.len(); - - // Avoid multiple allocations by using with_capacity - let mut new_incoming_keys = Vec::with_capacity(incoming_keys.len()); - let mut keys_to_fetch = Vec::new(); - let mut out_of_range_keys = Vec::new(); - - // Single pass filtering instead of multiple retain() calls - for (addr, record_type) in incoming_keys { - let key = addr.to_record_key(); - - // Skip if locally stored or already pending fetch - if locally_stored_keys.contains_key(&key) - || self - .to_be_fetched - .contains_key(&(key.clone(), record_type.clone(), holder)) - { - continue; - } - - // Check distance constraints - if let Some(farthest_distance) = self.farthest_acceptable_distance { - if self_address.distance(&addr) > farthest_distance { - out_of_range_keys.push(addr); - continue; - } - } - - new_incoming_keys.push((addr, record_type)); - } + let candidates = if is_fresh_replicate { + incoming_keys + .into_iter() + .map(|(addr, val_type)| (holder, addr, val_type)) + .collect() + } else { + self.valid_candidates(&holder, incoming_keys, locally_stored_keys, closest_k_peers) + }; // Remove any outdated entries in `to_be_fetched` self.remove_stored_keys(locally_stored_keys); - - // Special case for single new key - if new_incoming_keys.len() == 1 { - let (record_address, record_type) = new_incoming_keys[0].clone(); - - let new_data_key = (record_address.to_record_key(), record_type); - - if let Entry::Vacant(entry) = self.on_going_fetches.entry(new_data_key.clone()) { - let (record_key, _record_type) = new_data_key; - keys_to_fetch.push((holder, record_key)); - let _ = entry.insert((holder, Instant::now() + FETCH_TIMEOUT)); - } - - // To avoid later on un-necessary actions. - new_incoming_keys.clear(); - } - self.to_be_fetched .retain(|_, time_out| *time_out > Instant::now()); - let mut out_of_range_keys = vec![]; - // Filter out those out_of_range ones among the incoming_keys. - if let Some(ref distance_range) = self.distance_range { - new_incoming_keys.retain(|(addr, _record_type)| { - let is_in_range = - convert_distance_to_u256(&self_address.distance(addr)) <= *distance_range; - if !is_in_range { - out_of_range_keys.push(addr.clone()); - } - is_in_range - }); - } - - if !out_of_range_keys.is_empty() { - info!("Among {total_incoming_keys} incoming replications from {holder:?}, found {} out of range", out_of_range_keys.len()); - } - - // add in-range AND non existing keys to the fetcher - new_incoming_keys + let mut keys_to_fetch = vec![]; + // add valid, in-range AND non existing keys to the fetcher + candidates .into_iter() - .for_each(|(addr, record_type)| { - let _ = self - .to_be_fetched - .entry((addr.to_record_key(), record_type, holder)) - .or_insert(Instant::now() + PENDING_TIMEOUT); + .for_each(|(peer_id, addr, record_type)| { + if is_fresh_replicate { + // Fresh replicate shall always got prioritized. + let new_data_key = (addr.to_record_key(), record_type); + if let Entry::Vacant(entry) = self.on_going_fetches.entry(new_data_key) { + keys_to_fetch.push((holder, addr.to_record_key())); + let _ = entry.insert((holder, Instant::now() + FETCH_TIMEOUT)); + } + } else { + let _ = self + .to_be_fetched + .entry((addr.to_record_key(), record_type, peer_id)) + .or_insert(Instant::now() + PENDING_TIMEOUT); + } }); keys_to_fetch.extend(self.next_keys_to_fetch()); - keys_to_fetch } @@ -200,14 +163,10 @@ impl ReplicationFetcher { // Notify the replication fetcher about a newly added Record to the node. // The corresponding key can now be removed from the replication fetcher. // Also returns the next set of keys that has to be fetched from the peer/network. - // - // Note: for Register, which different content (i.e. record_type) bearing same record_key - // remove `on_going_fetches` entry bearing same `record_key` only, - // to avoid false FetchFailed alarm against the peer. pub(crate) fn notify_about_new_put( &mut self, new_put: RecordKey, - record_type: RecordType, + record_type: ValidationType, ) -> Vec<(PeerId, RecordKey)> { self.to_be_fetched .retain(|(key, t, _), _| key != &new_put || t != &record_type); @@ -218,11 +177,11 @@ impl ReplicationFetcher { self.next_keys_to_fetch() } - // An early completion of a fetch means the target is an old version record (Register or Spend). + // An early completion of a fetch means the target is an old version record pub(crate) fn notify_fetch_early_completed( &mut self, key_in: RecordKey, - record_type: RecordType, + record_type: ValidationType, ) -> Vec<(PeerId, RecordKey)> { self.to_be_fetched.retain(|(key, current_type, _), _| { if current_type == &record_type { @@ -327,6 +286,231 @@ impl ReplicationFetcher { .collect() } + // Record peers' healthy status after the storage chanllenge. + pub(crate) fn add_peer_scores(&mut self, scores: Vec<(PeerId, bool)>) { + for (peer_id, is_healthy) in scores { + let (peer_scores, last_seen) = self + .peers_scores + .entry(peer_id) + .or_insert((VecDeque::new(), Instant::now())); + peer_scores.push_back(is_healthy); + if peer_scores.len() > 2 { + let _ = peer_scores.pop_front(); + } + *last_seen = Instant::now(); + } + + // Once got enough scoring knowledge, the `majority` approach shall no longer be used. + if self.had_enough_scoring_knowledge() { + self.initial_replicates.clear(); + } + + // Pruning to avoid infinite growing, only keep the recent 20. + if self.peers_scores.len() > 20 { + let mut oldest_peer = PeerId::random(); + let mut oldest_timestamp = Instant::now(); + for (peer_id, (_peer_scores, last_seen)) in self.peers_scores.iter() { + if *last_seen < oldest_timestamp { + oldest_timestamp = *last_seen; + oldest_peer = *peer_id; + } + } + let _ = self.peers_scores.remove(&oldest_peer); + } + } + + // Among the incoming keys, figure out those: + // * not already stored + // * not on pending + // * within the range + // * from valid source peer + fn valid_candidates( + &mut self, + holder: &PeerId, + incoming_keys: Vec<(NetworkAddress, ValidationType)>, + locally_stored_keys: &HashMap, + closest_k_peers: Vec, + ) -> Vec<(PeerId, NetworkAddress, ValidationType)> { + match self.is_peer_trustworthy(holder) { + Some(true) => { + debug!("Replication source {holder:?} is trustworthy."); + let new_incoming_keys = self.in_range_new_keys( + holder, + incoming_keys, + locally_stored_keys, + closest_k_peers, + ); + new_incoming_keys + .into_iter() + .map(|(addr, val_type)| (*holder, addr, val_type)) + .collect() + } + Some(false) => { + debug!("Replication source {holder:?} is not trustworthy."); + vec![] + } + None => { + debug!("Not having enough network knowledge, using majority scheme instead."); + // Whenever we had enough scoring knowledge of peers, + // we shall no longer use the `majority copies` approach. + // This can prevent malicious neighbouring farming targeting existing nodes. + if self.had_enough_scoring_knowledge() { + // The replication source is probably a `new peer`. + // Just wait for the scoring knowledge to be built up. + return vec![]; + } + let new_incoming_keys = self.in_range_new_keys( + holder, + incoming_keys, + locally_stored_keys, + closest_k_peers, + ); + self.initial_majority_replicates(holder, new_incoming_keys) + } + } + } + + fn had_enough_scoring_knowledge(&self) -> bool { + self.peers_scores + .values() + .filter(|(scores, _last_seen)| scores.len() > 1) + .count() + >= CLOSE_GROUP_SIZE + } + + // Accumulates initial replicates when doesn't have enough knowledge of peers scores. + // Returns with entries that reached majority copies. + fn initial_majority_replicates( + &mut self, + holder: &PeerId, + incoming_keys: Vec<(NetworkAddress, ValidationType)>, + ) -> Vec<(PeerId, NetworkAddress, ValidationType)> { + let mut majorities = vec![]; + for addr_val_type in incoming_keys { + debug!( + "adding record {:?} from holder {holder:?} into initial accumulator", + addr_val_type.0 + ); + let peers = self + .initial_replicates + .entry(addr_val_type.clone()) + .or_default(); + let _ = peers.insert(*holder); + if peers.len() >= CLOSE_GROUP_SIZE / 2 { + majorities.push(addr_val_type); + } + } + + let mut result = vec![]; + for addr_val_type in majorities { + debug!("Accumulated majorities: {:?}", addr_val_type.0); + if let Some(peers) = self.initial_replicates.remove(&addr_val_type) { + for peer in peers { + result.push((peer, addr_val_type.0.clone(), addr_val_type.1.clone())); + } + } + } + + result + } + + // Among the incoming keys, figure out those: + // * not already stored + // * not on pending + // * within the range + fn in_range_new_keys( + &mut self, + holder: &PeerId, + incoming_keys: Vec<(NetworkAddress, ValidationType)>, + locally_stored_keys: &HashMap, + mut closest_k_peers: Vec, + ) -> Vec<(NetworkAddress, ValidationType)> { + // Pre-calculate self_address since it's used multiple times + let self_address = NetworkAddress::from_peer(self.self_peer_id); + closest_k_peers.push(self_address.clone()); + let total_incoming_keys = incoming_keys.len(); + + // Avoid multiple allocations by using with_capacity + let mut new_incoming_keys = Vec::with_capacity(incoming_keys.len()); + let mut out_of_range_keys = Vec::new(); + + // Single pass filtering instead of multiple retain() calls + for (addr, record_type) in incoming_keys { + let key = addr.to_record_key(); + + // Skip if locally stored or already pending fetch + if locally_stored_keys.contains_key(&key) + || self + .to_be_fetched + .contains_key(&(key.clone(), record_type.clone(), *holder)) + { + continue; + } + + // Check distance constraints + if let Some(farthest_distance) = self.farthest_acceptable_distance { + if self_address.distance(&addr) > farthest_distance { + out_of_range_keys.push(addr); + continue; + } + } + + new_incoming_keys.push((addr, record_type)); + } + + // Filter out those out_of_range ones among the incoming_keys. + if let Some(ref distance_range) = self.distance_range { + new_incoming_keys.retain(|(addr, _record_type)| { + let distance = &self_address.distance(addr); + debug!( + "Distance to target {addr:?} is {distance:?}, against range {distance_range:?}" + ); + let mut is_in_range = distance <= distance_range; + // For middle-range records, they could be farther than distance_range, + // but still supposed to be held by the closest group to us. + if !is_in_range && distance.0 - distance_range.0 < distance_range.0 { + closest_k_peers.sort_by_key(|key| key.distance(addr)); + let closest_group: HashSet<_> = closest_k_peers.iter().take(CLOSE_GROUP_SIZE).collect(); + if closest_group.contains(&self_address) { + debug!("Record {addr:?} has a far distance but still among {CLOSE_GROUP_SIZE} closest within {} neighbourd.", closest_k_peers.len()); + is_in_range = true; + } + } + if !is_in_range { + out_of_range_keys.push(addr.clone()); + } + is_in_range + }); + } + + if !out_of_range_keys.is_empty() && !new_incoming_keys.is_empty() { + info!("Among {total_incoming_keys} incoming replications from {holder:?}, {} new records and {} out of range", + new_incoming_keys.len(), out_of_range_keys.len()); + } + + new_incoming_keys + } + + // Check whether the peer is a trustworthy replication source. + // * Some(true) : peer is trustworthy + // * Some(false) : peer is not trustworthy + // * None : not having enough know to tell + fn is_peer_trustworthy(&self, holder: &PeerId) -> Option { + if let Some((scores, _last_seen)) = self.peers_scores.get(holder) { + if scores.len() > 1 { + let is_healthy = scores.iter().filter(|is_health| **is_health).count() > 1; + if !is_healthy { + info!("Peer {holder:?} is not a trustworthy replication source, as bearing scores of {scores:?}"); + } + Some(is_healthy) + } else { + None + } + } else { + None + } + } + // Just remove outdated entries in `on_going_fetch`, indicates a failure to fetch from network. // The node then considered to be in trouble and: // 1, the pending_entries from that node shall be removed from `to_be_fetched` list. @@ -344,19 +528,19 @@ impl ReplicationFetcher { } }); - let mut failed_holders = BTreeSet::new(); + let mut failed_holders = BTreeMap::new(); for (record_key, peer_id) in failed_fetches { - error!( - "Failed to fetch {:?} from {peer_id:?}", + debug!( + "Replication_fetcher has outdated fetch of {:?} from {peer_id:?}", PrettyPrintRecordKey::from(&record_key) ); - let _ = failed_holders.insert(peer_id); + let _ = failed_holders.insert(peer_id, record_key); } // now to clear any failed nodes from our lists. self.to_be_fetched - .retain(|(_, _, holder), _| !failed_holders.contains(holder)); + .retain(|(_, _, holder), _| !failed_holders.contains_key(holder)); // Such failed_hodlers (if any) shall be reported back and be excluded from RT. if !failed_holders.is_empty() { @@ -365,13 +549,13 @@ impl ReplicationFetcher { } /// Remove keys that we hold already and no longer need to be replicated. - /// This checks the hash on transactions to ensure we pull in divergent transactions. + /// This checks the hash on GraphEntry to ensure we pull in divergent GraphEntry. fn remove_stored_keys( &mut self, - existing_keys: &HashMap, + existing_keys: &HashMap, ) { self.to_be_fetched.retain(|(key, t, _), _| { - if let Some((_addr, record_type)) = existing_keys.get(key) { + if let Some((_addr, record_type, _data_type)) = existing_keys.get(key) { // check the address only against similar record types t != record_type } else { @@ -379,7 +563,7 @@ impl ReplicationFetcher { } }); self.on_going_fetches.retain(|(key, t), _| { - if let Some((_addr, record_type)) = existing_keys.get(key) { + if let Some((_addr, record_type, _data_type)) = existing_keys.get(key) { // check the address only against similar record types t != record_type } else { @@ -412,10 +596,14 @@ impl ReplicationFetcher { #[cfg(test)] mod tests { use super::{ReplicationFetcher, FETCH_TIMEOUT, MAX_PARALLEL_FETCH}; - use ant_protocol::{convert_distance_to_u256, storage::RecordType, NetworkAddress}; + use crate::CLOSE_GROUP_SIZE; + use ant_protocol::{storage::ValidationType, NetworkAddress}; use eyre::Result; use libp2p::{kad::RecordKey, PeerId}; - use std::{collections::HashMap, time::Duration}; + use std::{ + collections::{HashMap, HashSet}, + time::Duration, + }; use tokio::{sync::mpsc, time::sleep}; #[tokio::test] @@ -430,34 +618,53 @@ mod tests { (0..MAX_PARALLEL_FETCH * 2).for_each(|_| { let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key = NetworkAddress::from_record_key(&RecordKey::from(random_data)); - incoming_keys.push((key, RecordType::Chunk)); + incoming_keys.push((key, ValidationType::Chunk)); }); - let keys_to_fetch = - replication_fetcher.add_keys(PeerId::random(), incoming_keys, &locally_stored_keys); + let replication_src = PeerId::random(); + replication_fetcher.add_peer_scores(vec![(replication_src, true)]); + replication_fetcher.add_peer_scores(vec![(replication_src, true)]); + + let keys_to_fetch = replication_fetcher.add_keys( + replication_src, + incoming_keys, + &locally_stored_keys, + false, + vec![], + ); assert_eq!(keys_to_fetch.len(), MAX_PARALLEL_FETCH); + let replication_src_1 = PeerId::random(); + replication_fetcher.add_peer_scores(vec![(replication_src_1, true)]); + replication_fetcher.add_peer_scores(vec![(replication_src_1, true)]); // we should not fetch anymore keys let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key_1 = NetworkAddress::from_record_key(&RecordKey::from(random_data)); let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key_2 = NetworkAddress::from_record_key(&RecordKey::from(random_data)); let keys_to_fetch = replication_fetcher.add_keys( - PeerId::random(), - vec![(key_1, RecordType::Chunk), (key_2, RecordType::Chunk)], + replication_src_1, + vec![ + (key_1, ValidationType::Chunk), + (key_2, ValidationType::Chunk), + ], &locally_stored_keys, + false, + vec![], ); assert!(keys_to_fetch.is_empty()); - // List with length of 1 will be considered as `new data` and to be fetched immediately + // Fresh replication shall be fetched immediately let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key = NetworkAddress::from_record_key(&RecordKey::from(random_data)); let keys_to_fetch = replication_fetcher.add_keys( - PeerId::random(), - vec![(key, RecordType::Chunk)], + replication_src, + vec![(key, ValidationType::Chunk)], &locally_stored_keys, + true, + vec![], ); - assert!(!keys_to_fetch.is_empty()); + assert_eq!(keys_to_fetch.len(), 1); sleep(FETCH_TIMEOUT + Duration::from_secs(1)).await; @@ -483,24 +690,49 @@ mod tests { // Set distance range let distance_target = NetworkAddress::from_peer(PeerId::random()); let distance_range = self_address.distance(&distance_target); - let distance_256 = convert_distance_to_u256(&distance_range); - replication_fetcher.set_replication_distance_range(distance_256); + replication_fetcher.set_replication_distance_range(distance_range); + + let mut closest_k_peers = vec![]; + (0..19).for_each(|_| { + closest_k_peers.push(NetworkAddress::from_peer(PeerId::random())); + }); let mut incoming_keys = Vec::new(); let mut in_range_keys = 0; + let mut closest_k_peers_include_self = closest_k_peers.clone(); + closest_k_peers_include_self.push(self_address.clone()); (0..100).for_each(|_| { let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key = NetworkAddress::from_record_key(&RecordKey::from(random_data)); - if key.distance(&self_address) <= distance_range { + let distance = key.distance(&self_address); + if distance <= distance_range { in_range_keys += 1; + } else if distance.0 - distance_range.0 < distance_range.0 { + closest_k_peers_include_self.sort_by_key(|addr| key.distance(addr)); + let closest_group: HashSet<_> = closest_k_peers_include_self + .iter() + .take(CLOSE_GROUP_SIZE) + .collect(); + if closest_group.contains(&self_address) { + in_range_keys += 1; + } } - incoming_keys.push((key, RecordType::Chunk)); + incoming_keys.push((key, ValidationType::Chunk)); }); - let keys_to_fetch = - replication_fetcher.add_keys(PeerId::random(), incoming_keys, &Default::default()); + let replication_src = PeerId::random(); + replication_fetcher.add_peer_scores(vec![(replication_src, true)]); + replication_fetcher.add_peer_scores(vec![(replication_src, true)]); + + let keys_to_fetch = replication_fetcher.add_keys( + replication_src, + incoming_keys, + &Default::default(), + false, + closest_k_peers, + ); assert_eq!( keys_to_fetch.len(), replication_fetcher.on_going_fetches.len(), diff --git a/ant-networking/src/target_arch.rs b/ant-networking/src/target_arch.rs deleted file mode 100644 index 680528496a..0000000000 --- a/ant-networking/src/target_arch.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -#[cfg(not(target_arch = "wasm32"))] -pub use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -/// Wasm32 target arch does not support `time` or spawning via tokio -/// so we shim in alternatives here when building for that architecture -#[cfg(not(target_arch = "wasm32"))] -pub use tokio::{ - spawn, - time::{interval, sleep, timeout, Interval}, -}; - -#[cfg(target_arch = "wasm32")] -pub use std::time::Duration; - -#[cfg(target_arch = "wasm32")] -pub use wasmtimer::{ - std::{Instant, SystemTime, UNIX_EPOCH}, - tokio::{interval, sleep, timeout, Interval}, -}; - -#[cfg(target_arch = "wasm32")] -pub use wasm_bindgen_futures::spawn_local as spawn; diff --git a/ant-networking/src/time.rs b/ant-networking/src/time.rs new file mode 100644 index 0000000000..2bd0a9f043 --- /dev/null +++ b/ant-networking/src/time.rs @@ -0,0 +1,12 @@ +/// Copyright 2024 MaidSafe.net limited. +/// +/// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +/// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +/// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +/// KIND, either express or implied. Please review the Licences for the specific language governing +/// permissions and limitations relating to use of the SAFE Network Software. +pub use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub use tokio::{ + spawn, + time::{interval, sleep, timeout, Interval}, +}; diff --git a/ant-networking/src/transactions.rs b/ant-networking/src/transactions.rs deleted file mode 100644 index d4ab960971..0000000000 --- a/ant-networking/src/transactions.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{driver::GetRecordCfg, Network, NetworkError, Result}; -use ant_protocol::storage::{Transaction, TransactionAddress}; -use ant_protocol::{ - storage::{try_deserialize_record, RecordHeader, RecordKind, RetryStrategy}, - NetworkAddress, PrettyPrintRecordKey, -}; -use libp2p::kad::{Quorum, Record}; - -impl Network { - /// Gets Transactions at TransactionAddress from the Network. - pub async fn get_transactions(&self, address: TransactionAddress) -> Result> { - let key = NetworkAddress::from_transaction_address(address).to_record_key(); - let get_cfg = GetRecordCfg { - get_quorum: Quorum::All, - retry_strategy: Some(RetryStrategy::Quick), - target_record: None, - expected_holders: Default::default(), - is_register: false, - }; - let record = self.get_record_from_network(key.clone(), &get_cfg).await?; - debug!( - "Got record from the network, {:?}", - PrettyPrintRecordKey::from(&record.key) - ); - - get_transactions_from_record(&record) - } -} - -pub fn get_transactions_from_record(record: &Record) -> Result> { - let header = RecordHeader::from_record(record)?; - if let RecordKind::Transaction = header.kind { - let transactions = try_deserialize_record::>(record)?; - Ok(transactions) - } else { - warn!( - "RecordKind mismatch while trying to retrieve transactions from record {:?}", - PrettyPrintRecordKey::from(&record.key) - ); - Err(NetworkError::RecordKindMismatch(RecordKind::Transaction)) - } -} diff --git a/ant-networking/src/transport.rs b/ant-networking/src/transport.rs new file mode 100644 index 0000000000..63e0f83fc0 --- /dev/null +++ b/ant-networking/src/transport.rs @@ -0,0 +1,49 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +#[cfg(feature = "open-metrics")] +use crate::MetricsRegistries; +use libp2p::{ + core::{muxing::StreamMuxerBox, transport}, + identity::Keypair, + PeerId, Transport as _, +}; + +const MAX_STREAM_DATA_ENV_STR: &str = "ANT_MAX_STREAM_DATA"; + +pub(crate) fn build_transport( + keypair: &Keypair, + #[cfg(feature = "open-metrics")] registries: &mut MetricsRegistries, +) -> transport::Boxed<(PeerId, StreamMuxerBox)> { + let trans = generate_quic_transport(keypair); + #[cfg(feature = "open-metrics")] + let trans = libp2p::metrics::BandwidthTransport::new(trans, &mut registries.standard_metrics); + + let trans = trans.map(|(peer_id, muxer), _| (peer_id, StreamMuxerBox::new(muxer))); + + trans.boxed() +} + +fn generate_quic_transport( + keypair: &Keypair, +) -> libp2p::quic::GenTransport { + let mut quic_config = libp2p::quic::Config::new(keypair); + if let Ok(val) = std::env::var(MAX_STREAM_DATA_ENV_STR) { + match val.parse::() { + Ok(val) => { + quic_config.max_stream_data = val; + tracing::info!("Overriding QUIC connection receive window value to {val}"); + } + Err(e) => { + tracing::warn!("QUIC connection receive window value override failed. Could not parse `{MAX_STREAM_DATA_ENV_STR}={val}` as integer: {e}") + } + } + } + + libp2p::quic::tokio::Transport::new(quic_config) +} diff --git a/ant-networking/src/transport/mod.rs b/ant-networking/src/transport/mod.rs deleted file mode 100644 index 4f8b142993..0000000000 --- a/ant-networking/src/transport/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[cfg_attr(target_arch = "wasm32", path = "wasm32.rs")] -#[cfg_attr(not(target_arch = "wasm32"), path = "other.rs")] -pub(crate) mod mod_impl; - -pub(crate) use mod_impl::build_transport; diff --git a/ant-networking/src/transport/other.rs b/ant-networking/src/transport/other.rs deleted file mode 100644 index 75bca5ed27..0000000000 --- a/ant-networking/src/transport/other.rs +++ /dev/null @@ -1,26 +0,0 @@ -#[cfg(feature = "open-metrics")] -use crate::MetricsRegistries; -use libp2p::{ - core::{muxing::StreamMuxerBox, transport}, - identity::Keypair, - PeerId, Transport as _, -}; - -pub(crate) fn build_transport( - keypair: &Keypair, - #[cfg(feature = "open-metrics")] registries: &mut MetricsRegistries, -) -> transport::Boxed<(PeerId, StreamMuxerBox)> { - let trans = generate_quic_transport(keypair); - #[cfg(feature = "open-metrics")] - let trans = libp2p::metrics::BandwidthTransport::new(trans, &mut registries.standard_metrics); - - let trans = trans.map(|(peer_id, muxer), _| (peer_id, StreamMuxerBox::new(muxer))); - - trans.boxed() -} - -fn generate_quic_transport( - keypair: &Keypair, -) -> libp2p::quic::GenTransport { - libp2p::quic::tokio::Transport::new(libp2p::quic::Config::new(keypair)) -} diff --git a/ant-networking/src/transport/wasm32.rs b/ant-networking/src/transport/wasm32.rs deleted file mode 100644 index 5eb645063e..0000000000 --- a/ant-networking/src/transport/wasm32.rs +++ /dev/null @@ -1,18 +0,0 @@ -// wasm32 environments typically only support WebSockets (and WebRTC or WebTransport), so no plain UDP or TCP. - -use libp2p::{ - core::{muxing::StreamMuxerBox, transport, upgrade}, - identity::Keypair, - noise, websocket_websys, yamux, PeerId, Transport as _, -}; - -pub(crate) fn build_transport(keypair: &Keypair) -> transport::Boxed<(PeerId, StreamMuxerBox)> { - // We build a single transport here, WebSockets. - websocket_websys::Transport::default() - .upgrade(upgrade::Version::V1) - .authenticate( - noise::Config::new(keypair).expect("Signing libp2p-noise static DH keypair failed."), - ) - .multiplex(yamux::Config::default()) - .boxed() -} diff --git a/ant-node-manager/Cargo.toml b/ant-node-manager/Cargo.toml index e83f43dabb..07e986164b 100644 --- a/ant-node-manager/Cargo.toml +++ b/ant-node-manager/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-node-manager" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.11.7" +version = "0.11.8" [[bin]] name = "antctl" @@ -20,7 +20,6 @@ path = "src/bin/daemon/main.rs" [features] chaos = [] default = ["quic"] -local = [] nightly = [] open-metrics = [] otlp = [] @@ -30,20 +29,20 @@ tcp = [] websockets = [] [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } ant-releases = { version = "0.4.0" } -ant-service-management = { path = "../ant-service-management", version = "0.4.7" } +ant-service-management = { path = "../ant-service-management", version = "0.4.8" } chrono = "~0.4.19" clap = { version = "4.4.6", features = ["derive", "env"] } colored = "2.0.4" -color-eyre = "~0.6" +color-eyre = "0.6.3" dirs-next = "2.0.0" indicatif = { version = "0.17.5", features = ["tokio"] } -libp2p = { version = "0.54.1", features = [] } +libp2p = { version = "0.55.0", features = [] } libp2p-identity = { version = "0.2.7", features = ["rand"] } prost = { version = "0.9" } rand = "0.8.5" @@ -74,5 +73,5 @@ mockall = "0.12.1" reqwest = { version = "0.12", default-features = false, features = [ "json", "rustls-tls", -]} +] } predicates = "3.1.0" diff --git a/ant-node-manager/README.md b/ant-node-manager/README.md index e9147ac8be..b64d967aa9 100644 --- a/ant-node-manager/README.md +++ b/ant-node-manager/README.md @@ -6,9 +6,9 @@ It runs on Linux, macOS and Windows. ## Installation -The latest version can be installed via [safeup](https://github.com/maidsafe/safeup): +The latest version can be installed via [antup](https://github.com/maidsafe/antup): ``` -safeup antctl +antup antctl ``` A binary can also be obtained for your platform from the releases in this repository. @@ -356,7 +356,7 @@ faucet - RUNNING So by default, 25 node processes have been launched, along with a faucet. The faucet dispenses tokens for use when uploading files. We can now run `safe` commands against the local network. -The most common scenario for using a local network is for development, but you can also use it to exercise a lot of features locally. For more details, please see the 'Using a Local Network' section of the [main README](https://github.com/maidsafe/safe_network/tree/node-man-readme?tab=readme-ov-file#using-a-local-network). +The most common scenario for using a local network is for development, but you can also use it to exercise a lot of features locally. For more details, please see the 'Using a Local Network' section of the [main README](../README.md#using-a-local-network). Once you've finished, run `antctl local kill` to dispose the local network. @@ -366,9 +366,9 @@ Sometimes it will be necessary to run the integration tests in a local setup. Th The tests can be run from a VM, which is provided by a `Vagrantfile` in the `ant_node_manager` crate directory. The machine is defined to use libvirt rather than Virtualbox, so an installation of that is required, but that is beyond the scope of this document. -Assuming that you did have an installation of libvirt, you can get the VM by running `vagrant up`. Once the machine is available, run `vagrant ssh` to get a shell session inside it. For running the tests, switch to the root user using `sudo su -`. As part of the provisioning process, the current `safe_network` code is copied to the root user's home directory. To run the tests: +Assuming that you did have an installation of libvirt, you can get the VM by running `vagrant up`. Once the machine is available, run `vagrant ssh` to get a shell session inside it. For running the tests, switch to the root user using `sudo su -`. As part of the provisioning process, the current `autonomi` code is copied to the root user's home directory. To run the tests: ``` -cd safe_network +cd autonomi just node-man-integration-tests ``` diff --git a/ant-node-manager/src/add_services/config.rs b/ant-node-manager/src/add_services/config.rs index 7aac0eaeb6..b2037bbc86 100644 --- a/ant-node-manager/src/add_services/config.rs +++ b/ant-node-manager/src/add_services/config.rs @@ -85,7 +85,6 @@ pub struct InstallNodeServiceCtxBuilder { pub metrics_port: Option, pub node_ip: Option, pub node_port: Option, - pub owner: Option, pub peers_args: PeersArgs, pub rewards_address: RewardsAddress, pub rpc_socket_addr: SocketAddr, @@ -132,10 +131,6 @@ impl InstallNodeServiceCtxBuilder { args.push(OsString::from("--metrics-server-port")); args.push(OsString::from(metrics_port.to_string())); } - if let Some(owner) = self.owner { - args.push(OsString::from("--owner")); - args.push(OsString::from(owner)); - } if let Some(log_files) = self.max_archived_log_files { args.push(OsString::from("--max-archived-log-files")); args.push(OsString::from(log_files.to_string())); @@ -193,7 +188,6 @@ pub struct AddNodeServiceOptions { pub network_id: Option, pub node_ip: Option, pub node_port: Option, - pub owner: Option, pub peers_args: PeersArgs, pub rewards_address: RewardsAddress, pub rpc_address: Option, @@ -327,7 +321,6 @@ mod tests { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124") .unwrap(), @@ -363,7 +356,6 @@ mod tests { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124") .unwrap(), @@ -400,7 +392,6 @@ mod tests { network_id: Some(5), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124") .unwrap(), @@ -490,7 +481,6 @@ mod tests { builder.node_ip = Some(Ipv4Addr::new(192, 168, 1, 1)); builder.node_port = Some(12345); builder.metrics_port = Some(9090); - builder.owner = Some("test-owner".to_string()); builder.peers_args.addrs = vec![ "/ip4/127.0.0.1/tcp/8080".parse().unwrap(), "/ip4/192.168.1.1/tcp/8081".parse().unwrap(), @@ -531,8 +521,6 @@ mod tests { "12345", "--metrics-server-port", "9090", - "--owner", - "test-owner", "--max-archived-log-files", "10", "--max-log-files", diff --git a/ant-node-manager/src/add_services/mod.rs b/ant-node-manager/src/add_services/mod.rs index 76e8d46c12..842040b49c 100644 --- a/ant-node-manager/src/add_services/mod.rs +++ b/ant-node-manager/src/add_services/mod.rs @@ -78,16 +78,6 @@ pub async fn add_node( check_port_availability(port_option, &node_registry.nodes)?; } - let owner = match &options.owner { - Some(owner) => { - if owner.chars().any(|c| c.is_uppercase()) { - warn!("Owner name ({owner}) contains uppercase characters and will be converted to lowercase"); - } - Some(owner.to_lowercase()) - } - None => None, - }; - let antnode_file_name = options .antnode_src_path .file_name() @@ -213,7 +203,6 @@ pub async fn add_node( network_id: options.network_id, node_ip: options.node_ip, node_port, - owner: owner.clone(), peers_args: options.peers_args.clone(), rewards_address: options.rewards_address, rpc_socket_addr, @@ -254,7 +243,6 @@ pub async fn add_node( rewards_address: options.rewards_address, reward_balance: None, rpc_socket_addr, - owner: owner.clone(), peer_id: None, peers_args: options.peers_args.clone(), pid: None, diff --git a/ant-node-manager/src/add_services/tests.rs b/ant-node-manager/src/add_services/tests.rs index 58eaf31162..e1b2bea58a 100644 --- a/ant-node-manager/src/add_services/tests.rs +++ b/ant-node-manager/src/add_services/tests.rs @@ -142,7 +142,6 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), @@ -177,7 +176,6 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res network_id: None, node_ip: None, node_port: None, - owner: None, peers_args, rpc_address: None, rpc_port: None, @@ -300,7 +298,6 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: peers_args.clone(), pid: None, @@ -347,7 +344,6 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n network_id: None, node_ip: None, node_port: None, - owner: None, peers_args, rpc_address: Some(custom_rpc_address), rpc_port: None, @@ -437,7 +433,6 @@ async fn add_genesis_node_should_return_an_error_if_count_is_greater_than_1() -> network_id: None, node_ip: None, node_port: None, - owner: None, peers_args, rpc_address: None, rpc_port: None, @@ -534,7 +529,6 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), antnode_path: node_data_dir @@ -583,7 +577,6 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( name: "antnode2".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8083), @@ -632,7 +625,6 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( name: "antnode3".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8085), @@ -668,7 +660,6 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -815,7 +806,6 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 12001), @@ -850,7 +840,6 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -946,7 +935,6 @@ async fn add_new_node_should_add_another_service() -> Result<()> { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1006,7 +994,6 @@ async fn add_new_node_should_add_another_service() -> Result<()> { peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8083), - owner: None, antnode_path: node_data_dir .to_path_buf() .join("antnode2") @@ -1039,7 +1026,6 @@ async fn add_new_node_should_add_another_service() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -1203,7 +1189,6 @@ async fn add_node_should_create_service_file_with_first_arg() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -1360,7 +1345,6 @@ async fn add_node_should_create_service_file_with_peers_args() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -1512,7 +1496,6 @@ async fn add_node_should_create_service_file_with_local_arg() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -1668,7 +1651,6 @@ async fn add_node_should_create_service_file_with_network_contacts_url_arg() -> network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -1823,7 +1805,6 @@ async fn add_node_should_create_service_file_with_testnet_arg() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -1975,7 +1956,6 @@ async fn add_node_should_create_service_file_with_ignore_cache_arg() -> Result<( network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -2128,7 +2108,6 @@ async fn add_node_should_create_service_file_with_custom_bootstrap_cache_path() network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: peers_args.clone(), rpc_address: None, rpc_port: None, @@ -2274,7 +2253,6 @@ async fn add_node_should_create_service_file_with_network_id() -> Result<()> { network_id: Some(5), node_ip: None, node_port: None, - owner: None, peers_args: Default::default(), rpc_address: None, rpc_port: None, @@ -2418,7 +2396,6 @@ async fn add_node_should_use_custom_ip() -> Result<()> { network_id: None, node_ip: Some(custom_ip), node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -2516,7 +2493,6 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { name: "antnode1".to_string(), node_ip: None, node_port: Some(custom_port), - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 12001), @@ -2552,7 +2528,6 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { network_id: None, node_ip: None, node_port: Some(PortRange::Single(custom_port)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -2810,7 +2785,6 @@ async fn add_node_should_use_a_custom_port_range() -> Result<()> { network_id: None, node_ip: None, node_port: Some(PortRange::Range(12000, 12002)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -2886,7 +2860,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_port_is_used() -> R node_ip: None, node_port: Some(12000), number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -2931,7 +2904,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_port_is_used() -> R network_id: None, node_ip: None, node_port: Some(PortRange::Single(12000)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3004,7 +2976,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_port_in_range_is_us network_id: None, node_ip: None, node_port: Some(12000), - owner: None, peers_args: PeersArgs::default(), number: 1, peer_id: None, @@ -3050,7 +3021,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_port_in_range_is_us network_id: None, node_ip: None, node_port: Some(PortRange::Range(12000, 12002)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3129,7 +3099,6 @@ async fn add_node_should_return_an_error_if_port_and_node_count_do_not_match() - network_id: None, node_ip: None, node_port: Some(PortRange::Range(12000, 12002)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3213,7 +3182,6 @@ async fn add_node_should_return_an_error_if_multiple_services_are_specified_with network_id: None, node_ip: None, node_port: Some(PortRange::Single(12000)), - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3359,7 +3327,6 @@ async fn add_node_should_set_random_ports_if_enable_metrics_server_is_true() -> network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3497,7 +3464,6 @@ async fn add_node_should_set_max_archived_log_files() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3636,7 +3602,6 @@ async fn add_node_should_set_max_log_files() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3889,7 +3854,6 @@ async fn add_node_should_use_a_custom_port_range_for_metrics_server() -> Result< network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -3962,7 +3926,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_metrics_port_is_use node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -4007,7 +3970,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_metrics_port_is_use network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -4082,7 +4044,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_metrics_port_in_ran node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -4127,7 +4088,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_metrics_port_in_ran network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -4363,7 +4323,6 @@ async fn add_node_should_use_a_custom_port_range_for_the_rpc_server() -> Result< network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: Some(PortRange::Range(20000, 20002)), @@ -4447,7 +4406,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_rpc_port_is_used() node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -4492,7 +4450,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_rpc_port_is_used() network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: Some(PortRange::Single(8081)), @@ -4567,7 +4524,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_rpc_port_in_range_i node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -4612,7 +4568,6 @@ async fn add_node_should_return_an_error_if_duplicate_custom_rpc_port_in_range_i network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: Some(PortRange::Range(8081, 8082)), @@ -4708,7 +4663,6 @@ async fn add_node_should_disable_upnp_and_home_network_if_nat_status_is_public() name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 12001), @@ -4743,7 +4697,6 @@ async fn add_node_should_disable_upnp_and_home_network_if_nat_status_is_public() network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -4836,7 +4789,6 @@ async fn add_node_should_enable_upnp_if_nat_status_is_upnp() -> Result<()> { name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 12001), @@ -4871,7 +4823,6 @@ async fn add_node_should_enable_upnp_if_nat_status_is_upnp() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -4964,7 +4915,6 @@ async fn add_node_should_enable_home_network_if_nat_status_is_private() -> Resul name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 12001), @@ -4999,7 +4949,6 @@ async fn add_node_should_enable_home_network_if_nat_status_is_private() -> Resul network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -5086,7 +5035,6 @@ async fn add_node_should_return_an_error_if_nat_status_is_none_but_auto_set_nat_ network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -5708,7 +5656,6 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), @@ -5744,7 +5691,6 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -5838,7 +5784,6 @@ async fn add_node_should_apply_the_home_network_flag_if_it_is_used() -> Result<( name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), @@ -5874,7 +5819,6 @@ async fn add_node_should_apply_the_home_network_flag_if_it_is_used() -> Result<( network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -5968,7 +5912,6 @@ async fn add_node_should_add_the_node_in_user_mode() -> Result<()> { name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), @@ -6004,7 +5947,6 @@ async fn add_node_should_add_the_node_in_user_mode() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -6095,7 +6037,6 @@ async fn add_node_should_add_the_node_with_upnp_enabled() -> Result<()> { name: "antnode1".to_string(), node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rewards_address: RewardsAddress::from_str("0x03B770D9cD32077cC0bF330c13C114a87643B124")?, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), @@ -6131,7 +6072,6 @@ async fn add_node_should_add_the_node_with_upnp_enabled() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: None, peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, @@ -6168,145 +6108,6 @@ async fn add_node_should_add_the_node_with_upnp_enabled() -> Result<()> { Ok(()) } -#[tokio::test] -async fn add_node_should_assign_an_owner_in_lowercase() -> Result<()> { - let tmp_data_dir = assert_fs::TempDir::new()?; - let node_reg_path = tmp_data_dir.child("node_reg.json"); - - let latest_version = "0.96.4"; - let temp_dir = assert_fs::TempDir::new()?; - let node_data_dir = temp_dir.child("data"); - node_data_dir.create_dir_all()?; - let node_logs_dir = temp_dir.child("logs"); - node_logs_dir.create_dir_all()?; - let antnode_download_path = temp_dir.child(ANTNODE_FILE_NAME); - antnode_download_path.write_binary(b"fake antnode bin")?; - - let mut node_registry = NodeRegistry { - auditor: None, - daemon: None, - environment_variables: None, - faucet: None, - nat_status: None, - nodes: vec![], - save_path: node_reg_path.to_path_buf(), - }; - - let mut mock_service_control = MockServiceControl::new(); - let mut seq = Sequence::new(); - mock_service_control - .expect_get_available_port() - .times(1) - .returning(|| Ok(8081)) - .in_sequence(&mut seq); - - mock_service_control - .expect_install() - .with( - eq(ServiceInstallCtx { - args: vec![ - OsString::from("--rpc"), - OsString::from("127.0.0.1:8081"), - OsString::from("--root-dir"), - OsString::from( - node_data_dir - .to_path_buf() - .join("antnode1") - .to_string_lossy() - .to_string(), - ), - OsString::from("--log-output-dest"), - OsString::from( - node_logs_dir - .to_path_buf() - .join("antnode1") - .to_string_lossy() - .to_string(), - ), - OsString::from("--owner"), - OsString::from("discord_username"), - OsString::from("--rewards-address"), - OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), - OsString::from("evm-custom"), - OsString::from("--rpc-url"), - OsString::from("http://localhost:8545/"), - OsString::from("--payment-token-address"), - OsString::from("0x5FbDB2315678afecb367f032d93F642f64180aa3"), - OsString::from("--data-payments-address"), - OsString::from("0x8464135c8F25Da09e49BC8782676a84730C318bC"), - ], - autostart: false, - contents: None, - environment: None, - label: "antnode1".parse()?, - program: node_data_dir - .to_path_buf() - .join("antnode1") - .join(ANTNODE_FILE_NAME), - username: Some(get_username()), - working_directory: None, - }), - eq(false), - ) - .times(1) - .returning(|_, _| Ok(())) - .in_sequence(&mut seq); - - add_node( - AddNodeServiceOptions { - auto_restart: false, - auto_set_nat_flags: false, - count: None, - delete_antnode_src: true, - enable_metrics_server: false, - env_variables: None, - home_network: false, - log_format: None, - max_archived_log_files: None, - max_log_files: None, - metrics_port: None, - network_id: None, - node_ip: None, - node_port: None, - owner: Some("Discord_Username".to_string()), - peers_args: PeersArgs::default(), - rpc_address: None, - rpc_port: None, - antnode_dir_path: temp_dir.to_path_buf(), - antnode_src_path: antnode_download_path.to_path_buf(), - service_data_dir_path: node_data_dir.to_path_buf(), - service_log_dir_path: node_logs_dir.to_path_buf(), - upnp: false, - user: Some(get_username()), - user_mode: false, - version: latest_version.to_string(), - evm_network: EvmNetwork::Custom(CustomNetwork { - rpc_url_http: "http://localhost:8545".parse()?, - payment_token_address: RewardsAddress::from_str( - "0x5FbDB2315678afecb367f032d93F642f64180aa3", - )?, - data_payments_address: RewardsAddress::from_str( - "0x8464135c8F25Da09e49BC8782676a84730C318bC", - )?, - }), - rewards_address: RewardsAddress::from_str( - "0x03B770D9cD32077cC0bF330c13C114a87643B124", - )?, - }, - &mut node_registry, - &mock_service_control, - VerbosityLevel::Normal, - ) - .await?; - - assert_eq!( - node_registry.nodes[0].owner, - Some("discord_username".to_string()) - ); - - Ok(()) -} - #[tokio::test] async fn add_node_should_auto_restart() -> Result<()> { let tmp_data_dir = assert_fs::TempDir::new()?; @@ -6362,8 +6163,6 @@ async fn add_node_should_auto_restart() -> Result<()> { .to_string_lossy() .to_string(), ), - OsString::from("--owner"), - OsString::from("discord_username"), OsString::from("--rewards-address"), OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), OsString::from("evm-custom"), @@ -6407,7 +6206,6 @@ async fn add_node_should_auto_restart() -> Result<()> { network_id: None, node_ip: None, node_port: None, - owner: Some("discord_username".to_string()), peers_args: PeersArgs::default(), rpc_address: None, rpc_port: None, diff --git a/ant-node-manager/src/bin/cli/main.rs b/ant-node-manager/src/bin/cli/main.rs index b440cb09d8..5da5e99071 100644 --- a/ant-node-manager/src/bin/cli/main.rs +++ b/ant-node-manager/src/bin/cli/main.rs @@ -121,7 +121,7 @@ pub enum SubCmd { /// Useful to set log levels. Variables should be comma separated without spaces. /// /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] + #[clap(name = "env", long, use_value_delimiter = false, value_parser = parse_environment_variables)] env_variables: Option>, /// Specify what EVM network to use for payments. #[command(subcommand)] @@ -191,15 +191,6 @@ pub enum SubCmd { /// services, which in this case would be 5. The range must also go from lower to higher. #[clap(long, value_parser = PortRange::parse)] node_port: Option, - /// Specify the owner for the node service. - /// - /// This is mainly used for the 'Beta Rewards' programme, for linking your Discord username - /// to the node. - /// - /// If the option is not used, the node will assign its own username and the service will - /// run as normal. - #[clap(long)] - owner: Option, /// Provide a path for the antnode binary to be used by the service. /// /// Useful for creating the service using a custom built binary. @@ -255,8 +246,6 @@ pub enum SubCmd { #[clap(long)] version: Option, }, - #[clap(subcommand)] - Auditor(AuditorSubCmd), /// Get node reward balances. #[clap(name = "balance")] Balance { @@ -426,7 +415,7 @@ pub enum SubCmd { /// spaces. /// /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] + #[clap(name = "env", long, use_value_delimiter = false, value_parser = parse_environment_variables)] env_variables: Option>, /// Set this flag to force the upgrade command to replace binaries without comparing any /// version numbers. @@ -468,110 +457,6 @@ pub enum SubCmd { }, } -/// Manage the Auditor service. -#[derive(Subcommand, Debug)] -pub enum AuditorSubCmd { - /// Add an auditor service to collect and verify Spends from the network. - /// - /// By default, the latest sn_auditor binary will be downloaded; however, it is possible to - /// provide a binary either by specifying a URL, a local path, or a specific version number. - /// - /// This command must run as the root/administrative user. - #[clap(name = "add")] - Add { - /// Secret encryption key of the beta rewards to decypher - /// discord usernames of the beta participants - #[clap(short = 'k', long, value_name = "hex_secret_key")] - beta_encryption_key: Option, - /// Provide environment variables for the auditor service. - /// - /// Useful to set log levels. Variables should be comma separated without spaces. - /// - /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] - env_variables: Option>, - /// Provide the path for the log directory for the auditor. - /// - /// If not provided, the default location /var/log/auditor. - #[clap(long, verbatim_doc_comment)] - log_dir_path: Option, - /// Provide a path for the auditor binary to be used by the service. - /// - /// Useful for creating the auditor service using a custom built binary. - #[clap(long)] - path: Option, - #[command(flatten)] - peers: Box, - /// Provide a auditor binary using a URL. - /// - /// The binary must be inside a zip or gzipped tar archive. - /// - /// This option can be used to test a auditor binary that has been built from a forked - /// branch and uploaded somewhere. A typical use case would be for a developer who launches - /// a testnet to test some changes they have on a fork. - #[clap(long, conflicts_with = "version")] - url: Option, - /// Provide a specific version of the auditor to be installed. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - /// - /// The binary will be downloaded. - #[clap(long)] - version: Option, - }, - /// Start the auditor service. - /// - /// This command must run as the root/administrative user. - #[clap(name = "start")] - Start {}, - /// Stop the auditor service. - /// - /// This command must run as the root/administrative user. - #[clap(name = "stop")] - Stop {}, - /// Upgrade the Auditor. - /// - /// The running auditor will be stopped, its binary will be replaced, then it will be started - /// again. - /// - /// This command must run as the root/administrative user. - #[clap(name = "upgrade")] - Upgrade { - /// Set this flag to upgrade the auditor without starting it. - /// - /// Can be useful for testing scenarios. - #[clap(long)] - do_not_start: bool, - /// Set this flag to force the upgrade command to replace binaries without comparing any - /// version numbers. - /// - /// Required if we want to downgrade, or for testing purposes. - #[clap(long)] - force: bool, - /// Provide environment variables for the auditor service. - /// - /// Values set when the service was added will be overridden. - /// - /// Useful to set log levels. Variables should be comma separated without spaces. - /// - /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] - env_variables: Option>, - /// Provide a binary to upgrade to using a URL. - /// - /// The binary must be inside a zip or gzipped tar archive. - /// - /// This can be useful for testing scenarios. - #[clap(long, conflicts_with = "version")] - url: Option, - /// Upgrade to a specific version rather than the latest version. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - #[clap(long)] - version: Option, - }, -} - /// Manage the RPC service. #[derive(Subcommand, Debug)] pub enum DaemonSubCmd { @@ -595,7 +480,7 @@ pub enum DaemonSubCmd { /// Useful to set log levels. Variables should be comma separated without spaces. /// /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] + #[clap(name = "env", long, use_value_delimiter = false, value_parser = parse_environment_variables)] env_variables: Option>, /// Specify a port for the daemon to listen on. #[clap(long, default_value_t = 12500)] @@ -652,7 +537,7 @@ pub enum FaucetSubCmd { /// Useful to set log levels. Variables should be comma separated without spaces. /// /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] + #[clap(name = "env", long, use_value_delimiter = false, value_parser = parse_environment_variables)] env_variables: Option>, /// Provide the path for the log directory for the faucet. /// @@ -719,7 +604,7 @@ pub enum FaucetSubCmd { /// Useful to set log levels. Variables should be comma separated without spaces. /// /// Example: --env ANT_LOG=all,RUST_LOG=libp2p=debug - #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] + #[clap(name = "env", long, use_value_delimiter = false, value_parser = parse_environment_variables)] env_variables: Option>, /// Provide a binary to upgrade to using a URL. /// @@ -796,7 +681,7 @@ pub enum LocalSubCmd { Join { /// Set to build the antnode and faucet binaries. /// - /// This option requires the command run from the root of the safe_network repository. + /// This option requires the command run from the root of the autonomi repository. #[clap(long)] build: bool, /// The number of nodes to run. @@ -862,19 +747,6 @@ pub enum LocalSubCmd { node_version: Option, #[command(flatten)] peers: PeersArgs, - /// Specify the owner for each node in the local network - /// - /// The argument exists to support testing scenarios. - #[clap(long, conflicts_with = "owner_prefix")] - owner: Option, - /// Use this argument to launch each node in the network with an individual owner. - /// - /// Assigned owners will take the form "prefix_1", "prefix_2" etc., where "prefix" will be - /// replaced by the value specified by this argument. - /// - /// The argument exists to support testing scenarios. - #[clap(long, conflicts_with = "owner")] - owner_prefix: Option, /// Specify a port for the RPC service(s). /// /// If not used, ports will be selected at random. @@ -905,7 +777,7 @@ pub enum LocalSubCmd { Run { /// Set to build the antnode and faucet binaries. /// - /// This option requires the command run from the root of the safe_network repository. + /// This option requires the command run from the root of the autonomi repository. #[clap(long)] build: bool, /// Set to remove the client data directory and kill any existing local network. @@ -972,20 +844,6 @@ pub enum LocalSubCmd { /// The version and path arguments are mutually exclusive. #[clap(long, conflicts_with = "build")] node_version: Option, - /// Specify the owner for each node in the local network - /// - /// The argument exists to support testing scenarios. - #[clap(long, conflicts_with = "owner_prefix")] - owner: Option, - /// Use this argument to launch each node in the network with an individual owner. - /// - /// Assigned owners will take the form "prefix_1", "prefix_2" etc., where "prefix" will be - /// replaced by the value specified by this argument. - /// - /// The argument exists to support testing scenarios. - #[clap(long)] - #[clap(long, conflicts_with = "owner")] - owner_prefix: Option, /// Specify a port for the RPC service(s). /// /// If not used, ports will be selected at random. @@ -1083,7 +941,6 @@ async fn main() -> Result<()> { network_id, node_ip, node_port, - owner, path, peers, rewards_address, @@ -1111,7 +968,6 @@ async fn main() -> Result<()> { network_id, node_ip, node_port, - owner, peers, rewards_address, rpc_address, @@ -1126,38 +982,6 @@ async fn main() -> Result<()> { .await?; Ok(()) } - Some(SubCmd::Auditor(AuditorSubCmd::Add { - beta_encryption_key, - env_variables, - log_dir_path, - path, - peers, - url, - version, - })) => { - cmd::auditor::add( - beta_encryption_key, - env_variables, - log_dir_path, - *peers, - path, - url, - version, - verbosity, - ) - .await - } - Some(SubCmd::Auditor(AuditorSubCmd::Start {})) => cmd::auditor::start(verbosity).await, - Some(SubCmd::Auditor(AuditorSubCmd::Stop {})) => cmd::auditor::stop(verbosity).await, - Some(SubCmd::Auditor(AuditorSubCmd::Upgrade { - do_not_start, - force, - env_variables, - url, - version, - })) => { - cmd::auditor::upgrade(do_not_start, force, env_variables, url, version, verbosity).await - } Some(SubCmd::Balance { peer_id: peer_ids, service_name: service_names, @@ -1223,8 +1047,6 @@ async fn main() -> Result<()> { node_port, node_version, log_format, - owner, - owner_prefix, peers, rpc_port, rewards_address, @@ -1246,8 +1068,6 @@ async fn main() -> Result<()> { node_port, node_version, log_format, - owner, - owner_prefix, peers, rpc_port, rewards_address, @@ -1269,8 +1089,6 @@ async fn main() -> Result<()> { node_path, node_port, node_version, - owner, - owner_prefix, rpc_port, rewards_address, evm_network, @@ -1292,8 +1110,6 @@ async fn main() -> Result<()> { node_port, node_version, log_format, - owner, - owner_prefix, rpc_port, rewards_address, evm_network, @@ -1410,18 +1226,18 @@ fn parse_environment_variables(env_var: &str) -> Result<(String, String)> { async fn configure_winsw(verbosity: VerbosityLevel) -> Result<()> { use ant_node_manager::config::get_node_manager_path; - // If the node manager was installed using `safeup`, it would have put the winsw.exe binary at + // If the node manager was installed using `antup`, it would have put the winsw.exe binary at // `C:\Users\\autonomi\winsw.exe`, sitting it alongside the other safe-related binaries. // // However, if the node manager has been obtained by other means, we can put winsw.exe // alongside the directory where the services are defined. This prevents creation of what would // seem like a random `autonomi` directory in the user's home directory. - let safeup_winsw_path = dirs_next::home_dir() + let antup_winsw_path = dirs_next::home_dir() .ok_or_else(|| eyre!("Could not obtain user home directory"))? .join("autonomi") .join("winsw.exe"); - if safeup_winsw_path.exists() { - ant_node_manager::helpers::configure_winsw(&safeup_winsw_path, verbosity).await?; + if antup_winsw_path.exists() { + ant_node_manager::helpers::configure_winsw(&antup_winsw_path, verbosity).await?; } else { ant_node_manager::helpers::configure_winsw( &get_node_manager_path()?.join("winsw.exe"), diff --git a/ant-node-manager/src/bin/cli/subcommands/evm_network.rs b/ant-node-manager/src/bin/cli/subcommands/evm_network.rs index 2d795846cf..59d6078ad3 100644 --- a/ant-node-manager/src/bin/cli/subcommands/evm_network.rs +++ b/ant-node-manager/src/bin/cli/subcommands/evm_network.rs @@ -6,9 +6,9 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_evm::{utils::get_evm_network_from_env, EvmNetwork}; +use ant_evm::{get_evm_network, EvmNetwork}; use clap::Subcommand; -use color_eyre::{eyre::Result, Section}; +use color_eyre::Result; #[derive(Subcommand, Clone, Debug)] #[allow(clippy::enum_variant_names)] @@ -19,6 +19,9 @@ pub enum EvmNetworkCommand { /// Use the Arbitrum Sepolia network EvmArbitrumSepolia, + /// Use the Arbitrum Sepolia network with test contracts + EvmArbitrumSepoliaTest, + /// Use a custom network EvmCustom { /// The RPC URL for the custom network @@ -45,14 +48,9 @@ impl TryInto for EvmNetworkCommand { match self { Self::EvmArbitrumOne => Ok(EvmNetwork::ArbitrumOne), Self::EvmArbitrumSepolia => Ok(EvmNetwork::ArbitrumSepolia), + Self::EvmArbitrumSepoliaTest => Ok(EvmNetwork::ArbitrumSepoliaTest), Self::EvmLocal => { - if !cfg!(feature = "local") { - return Err(color_eyre::eyre::eyre!( - "The 'local' feature flag is not enabled." - )) - .suggestion("Enable the 'local' feature flag to use the local EVM testnet."); - } - let network = get_evm_network_from_env()?; + let network = get_evm_network(true)?; Ok(network) } Self::EvmCustom { diff --git a/ant-node-manager/src/cmd/local.rs b/ant-node-manager/src/cmd/local.rs index 2f0b3b465b..736f7806f7 100644 --- a/ant-node-manager/src/cmd/local.rs +++ b/ant-node-manager/src/cmd/local.rs @@ -34,8 +34,6 @@ pub async fn join( node_port: Option, node_version: Option, log_format: Option, - owner: Option, - owner_prefix: Option, _peers_args: PeersArgs, rpc_port: Option, rewards_address: RewardsAddress, @@ -78,8 +76,6 @@ pub async fn join( metrics_port, node_count: count, node_port, - owner, - owner_prefix, peers: None, rpc_port, skip_validation, @@ -119,8 +115,6 @@ pub async fn run( node_port: Option, node_version: Option, log_format: Option, - owner: Option, - owner_prefix: Option, rpc_port: Option, rewards_address: RewardsAddress, evm_network: Option, @@ -188,8 +182,6 @@ pub async fn run( metrics_port, node_port, node_count: count, - owner, - owner_prefix, peers: None, rpc_port, skip_validation, diff --git a/ant-node-manager/src/cmd/mod.rs b/ant-node-manager/src/cmd/mod.rs index 45138e640d..20a25fd99a 100644 --- a/ant-node-manager/src/cmd/mod.rs +++ b/ant-node-manager/src/cmd/mod.rs @@ -181,9 +181,6 @@ fn build_binary(bin_type: &ReleaseType) -> Result { if cfg!(feature = "otlp") { args.extend(["--features", "otlp"]); } - if cfg!(feature = "local") { - args.extend(["--features", "local"]); - } if cfg!(feature = "websockets") { args.extend(["--features", "websockets"]); } diff --git a/ant-node-manager/src/cmd/node.rs b/ant-node-manager/src/cmd/node.rs index 5ab42c0ea8..3812834811 100644 --- a/ant-node-manager/src/cmd/node.rs +++ b/ant-node-manager/src/cmd/node.rs @@ -52,7 +52,6 @@ pub async fn add( network_id: Option, node_ip: Option, node_port: Option, - owner: Option, mut peers_args: PeersArgs, rewards_address: RewardsAddress, rpc_address: Option, @@ -129,7 +128,6 @@ pub async fn add( network_id, node_ip, node_port, - owner, peers_args, rewards_address, rpc_address, @@ -598,7 +596,6 @@ pub async fn maintain_n_running_nodes( network_id: Option, node_ip: Option, node_port: Option, - owner: Option, peers_args: PeersArgs, rewards_address: RewardsAddress, rpc_address: Option, @@ -703,7 +700,6 @@ pub async fn maintain_n_running_nodes( network_id, node_ip, Some(PortRange::Single(port)), - owner.clone(), peers_args.clone(), rewards_address, rpc_address, diff --git a/ant-node-manager/src/lib.rs b/ant-node-manager/src/lib.rs index 8b2aaee95b..e0d6d908d3 100644 --- a/ant-node-manager/src/lib.rs +++ b/ant-node-manager/src/lib.rs @@ -771,7 +771,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -885,7 +884,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -964,7 +962,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1083,7 +1080,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1175,7 +1171,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1277,7 +1272,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1378,7 +1372,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1449,7 +1442,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1512,7 +1504,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1573,7 +1564,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1637,7 +1627,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: None, peers_args: PeersArgs::default(), pid: None, @@ -1712,7 +1701,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1852,7 +1840,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -1953,7 +1940,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2099,7 +2085,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2257,7 +2242,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2410,7 +2394,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2564,7 +2547,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2743,7 +2725,6 @@ mod tests { node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -2915,7 +2896,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3093,7 +3073,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3254,7 +3233,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3424,7 +3402,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3604,7 +3581,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3779,7 +3755,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -3949,7 +3924,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -4129,7 +4103,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -4290,7 +4263,6 @@ network_id: None, network_id: None, node_ip: None, node_port: None, - owner: None, number: 1, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", @@ -4456,7 +4428,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -4618,7 +4589,6 @@ network_id: None, number: 1, node_ip: Some(Ipv4Addr::new(192, 168, 1, 1)), node_port: None, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -4783,7 +4753,6 @@ network_id: None, number: 1, node_ip: None, node_port: Some(12000), - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -4944,7 +4913,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -5109,7 +5077,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -5272,7 +5239,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -5437,7 +5403,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -5483,171 +5448,6 @@ network_id: None, Ok(()) } - #[tokio::test] - async fn upgrade_should_retain_owner() -> Result<()> { - let current_version = "0.1.0"; - let target_version = "0.2.0"; - - let tmp_data_dir = assert_fs::TempDir::new()?; - let current_install_dir = tmp_data_dir.child("antnode_install"); - current_install_dir.create_dir_all()?; - - let current_node_bin = current_install_dir.child("antnode"); - current_node_bin.write_binary(b"fake antnode binary")?; - let target_node_bin = tmp_data_dir.child("antnode"); - target_node_bin.write_binary(b"fake antnode binary")?; - - let mut mock_service_control = MockServiceControl::new(); - let mut mock_rpc_client = MockRpcClient::new(); - - // before binary upgrade - mock_service_control - .expect_get_process_pid() - .with(eq(current_node_bin.to_path_buf().clone())) - .times(1) - .returning(|_| Ok(1000)); - mock_service_control - .expect_stop() - .with(eq("antnode1"), eq(false)) - .times(1) - .returning(|_, _| Ok(())); - - // after binary upgrade - mock_service_control - .expect_uninstall() - .with(eq("antnode1"), eq(false)) - .times(1) - .returning(|_, _| Ok(())); - mock_service_control - .expect_install() - .with( - eq(ServiceInstallCtx { - args: vec![ - OsString::from("--rpc"), - OsString::from("127.0.0.1:8081"), - OsString::from("--root-dir"), - OsString::from("/var/antctl/services/antnode1"), - OsString::from("--log-output-dest"), - OsString::from("/var/log/antnode/antnode1"), - OsString::from("--owner"), - OsString::from("discord_username"), - OsString::from("--rewards-address"), - OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), - OsString::from("evm-arbitrum-one"), - ], - autostart: false, - contents: None, - environment: None, - label: "antnode1".parse()?, - program: current_node_bin.to_path_buf(), - username: Some("ant".to_string()), - working_directory: None, - }), - eq(false), - ) - .times(1) - .returning(|_, _| Ok(())); - - // after service restart - mock_service_control - .expect_start() - .with(eq("antnode1"), eq(false)) - .times(1) - .returning(|_, _| Ok(())); - mock_service_control - .expect_wait() - .with(eq(3000)) - .times(1) - .returning(|_| ()); - mock_service_control - .expect_get_process_pid() - .with(eq(current_node_bin.to_path_buf().clone())) - .times(1) - .returning(|_| Ok(100)); - - mock_rpc_client.expect_node_info().times(1).returning(|| { - Ok(NodeInfo { - pid: 2000, - peer_id: PeerId::from_str("12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR")?, - data_path: PathBuf::from("/var/antctl/services/antnode1"), - log_path: PathBuf::from("/var/log/antnode/antnode1"), - version: target_version.to_string(), - uptime: std::time::Duration::from_secs(1), // the service was just started - wallet_balance: 0, - }) - }); - mock_rpc_client - .expect_network_info() - .times(1) - .returning(|| { - Ok(NetworkInfo { - connected_peers: Vec::new(), - listeners: Vec::new(), - }) - }); - - let mut service_data = NodeServiceData { - auto_restart: false, - connected_peers: None, - data_dir_path: PathBuf::from("/var/antctl/services/antnode1"), - evm_network: EvmNetwork::ArbitrumOne, - home_network: false, - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/antnode/antnode1"), - log_format: None, - max_archived_log_files: None, - max_log_files: None, - metrics_port: None, - network_id: None, - node_ip: None, - node_port: None, - number: 1, - owner: Some("discord_username".to_string()), - peer_id: Some(PeerId::from_str( - "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", - )?), - peers_args: PeersArgs::default(), - pid: Some(1000), - rewards_address: RewardsAddress::from_str( - "0x03B770D9cD32077cC0bF330c13C114a87643B124", - )?, - reward_balance: Some(AttoTokens::zero()), - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - antnode_path: current_node_bin.to_path_buf(), - service_name: "antnode1".to_string(), - status: ServiceStatus::Running, - upnp: false, - user: Some("ant".to_string()), - user_mode: false, - version: current_version.to_string(), - }; - let service = NodeService::new(&mut service_data, Box::new(mock_rpc_client)); - - let mut service_manager = ServiceManager::new( - service, - Box::new(mock_service_control), - VerbosityLevel::Normal, - ); - - service_manager - .upgrade(UpgradeOptions { - auto_restart: false, - env_variables: None, - force: false, - start_service: true, - target_bin_path: target_node_bin.to_path_buf(), - target_version: Version::parse(target_version).unwrap(), - }) - .await?; - - assert_eq!( - service_manager.service.service_data.owner, - Some("discord_username".to_string()) - ); - - Ok(()) - } - #[tokio::test] async fn upgrade_should_retain_auto_restart() -> Result<()> { let current_version = "0.1.0"; @@ -5694,8 +5494,6 @@ network_id: None, OsString::from("/var/antctl/services/antnode1"), OsString::from("--log-output-dest"), OsString::from("/var/log/antnode/antnode1"), - OsString::from("--owner"), - OsString::from("discord_username"), OsString::from("--rewards-address"), OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), OsString::from("evm-arbitrum-one"), @@ -5767,7 +5565,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: Some("discord_username".to_string()), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -5856,8 +5653,6 @@ network_id: None, OsString::from("/var/antctl/services/antnode1"), OsString::from("--log-output-dest"), OsString::from("/var/log/antnode/antnode1"), - OsString::from("--owner"), - OsString::from("discord_username"), OsString::from("--rewards-address"), OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), OsString::from("evm-custom"), @@ -5943,7 +5738,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: Some("discord_username".to_string()), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -6033,8 +5827,6 @@ network_id: None, OsString::from("/var/antctl/services/antnode1"), OsString::from("--log-output-dest"), OsString::from("/var/log/antnode/antnode1"), - OsString::from("--owner"), - OsString::from("discord_username"), OsString::from("--rewards-address"), OsString::from("0x03B770D9cD32077cC0bF330c13C114a87643B124"), OsString::from("evm-custom"), @@ -6120,7 +5912,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: Some("discord_username".to_string()), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -6285,7 +6076,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), @@ -6370,7 +6160,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peers_args: PeersArgs::default(), peer_id: None, pid: None, @@ -6439,7 +6228,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peers_args: PeersArgs::default(), pid: Some(1000), peer_id: Some(PeerId::from_str( @@ -6523,7 +6311,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, peers_args: PeersArgs::default(), pid: Some(1000), peer_id: Some(PeerId::from_str( @@ -6602,7 +6389,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, pid: None, peers_args: PeersArgs::default(), peer_id: None, @@ -6679,7 +6465,6 @@ network_id: None, node_ip: None, node_port: None, number: 1, - owner: None, pid: None, peers_args: PeersArgs::default(), peer_id: None, diff --git a/ant-node-manager/src/local.rs b/ant-node-manager/src/local.rs index 6acd1d6531..a7ed6529cb 100644 --- a/ant-node-manager/src/local.rs +++ b/ant-node-manager/src/local.rs @@ -43,7 +43,6 @@ pub trait Launcher { log_format: Option, metrics_port: Option, node_port: Option, - owner: Option, rpc_socket_addr: SocketAddr, rewards_address: RewardsAddress, evm_network: Option, @@ -67,18 +66,12 @@ impl Launcher for LocalSafeLauncher { log_format: Option, metrics_port: Option, node_port: Option, - owner: Option, rpc_socket_addr: SocketAddr, rewards_address: RewardsAddress, evm_network: Option, ) -> Result<()> { let mut args = Vec::new(); - if let Some(owner) = owner { - args.push("--owner".to_string()); - args.push(owner); - } - if first { args.push("--first".to_string()) } @@ -218,8 +211,6 @@ pub struct LocalNetworkOptions { pub metrics_port: Option, pub node_port: Option, pub node_count: u16, - pub owner: Option, - pub owner_prefix: Option, pub peers: Option>, pub rpc_port: Option, pub skip_validation: bool, @@ -289,7 +280,6 @@ pub async fn run_network( let rpc_client = RpcClient::from_socket_addr(rpc_socket_addr); let number = (node_registry.nodes.len() as u16) + 1; - let owner = get_node_owner(&options.owner_prefix, &options.owner, &number); let node = run_node( RunNodeOptions { first: true, @@ -298,7 +288,6 @@ pub async fn run_network( interval: options.interval, log_format: options.log_format, number, - owner, rpc_socket_addr, rewards_address: options.rewards_address, evm_network: options.evm_network.clone(), @@ -337,7 +326,6 @@ pub async fn run_network( let rpc_client = RpcClient::from_socket_addr(rpc_socket_addr); let number = (node_registry.nodes.len() as u16) + 1; - let owner = get_node_owner(&options.owner_prefix, &options.owner, &number); let node = run_node( RunNodeOptions { first: false, @@ -346,7 +334,6 @@ pub async fn run_network( interval: options.interval, log_format: options.log_format, number, - owner, rpc_socket_addr, rewards_address: options.rewards_address, evm_network: options.evm_network.clone(), @@ -386,7 +373,6 @@ pub struct RunNodeOptions { pub metrics_port: Option, pub node_port: Option, pub number: u16, - pub owner: Option, pub rpc_socket_addr: SocketAddr, pub rewards_address: RewardsAddress, pub evm_network: Option, @@ -405,7 +391,6 @@ pub async fn run_node( run_options.log_format, run_options.metrics_port, run_options.node_port, - run_options.owner.clone(), run_options.rpc_socket_addr, run_options.rewards_address, run_options.evm_network.clone(), @@ -439,7 +424,6 @@ pub async fn run_node( node_ip: None, node_port: run_options.node_port, number: run_options.number, - owner: run_options.owner, peer_id: Some(peer_id), peers_args: PeersArgs { first: run_options.first, @@ -513,18 +497,6 @@ async fn validate_network(node_registry: &mut NodeRegistry, peers: Vec, - owner: &Option, - number: &u16, -) -> Option { - if let Some(prefix) = owner_prefix { - Some(format!("{}_{}", prefix, number)) - } else { - owner.clone() - } -} - #[cfg(test)] mod tests { use super::*; @@ -569,13 +541,12 @@ mod tests { eq(None), eq(None), eq(None), - eq(None), eq(rpc_socket_addr), eq(rewards_address), eq(None), ) .times(1) - .returning(|_, _, _, _, _, _, _, _| Ok(())); + .returning(|_, _, _, _, _, _, _| Ok(())); mock_launcher .expect_wait() .with(eq(100)) @@ -618,7 +589,6 @@ mod tests { metrics_port: None, node_port: None, number: 1, - owner: None, rpc_socket_addr, rewards_address, evm_network: None, diff --git a/ant-node-manager/src/rpc.rs b/ant-node-manager/src/rpc.rs index 1af38833ff..c47a0927ba 100644 --- a/ant-node-manager/src/rpc.rs +++ b/ant-node-manager/src/rpc.rs @@ -77,7 +77,6 @@ pub async fn restart_node_service( network_id: current_node_clone.network_id, node_ip: current_node_clone.node_ip, node_port: current_node_clone.get_antnode_port(), - owner: current_node_clone.owner.clone(), peers_args: current_node_clone.peers_args.clone(), rewards_address: current_node_clone.rewards_address, rpc_socket_addr: current_node_clone.rpc_socket_addr, @@ -193,7 +192,6 @@ pub async fn restart_node_service( network_id: current_node_clone.network_id, node_ip: current_node_clone.node_ip, node_port: None, - owner: None, peers_args: current_node_clone.peers_args.clone(), rewards_address: current_node_clone.rewards_address, rpc_socket_addr: current_node_clone.rpc_socket_addr, @@ -223,7 +221,6 @@ pub async fn restart_node_service( node_ip: current_node_clone.node_ip, node_port: None, number: new_node_number as u16, - owner: None, peer_id: None, peers_args: current_node_clone.peers_args.clone(), pid: None, diff --git a/ant-node-rpc-client/Cargo.toml b/ant-node-rpc-client/Cargo.toml index 80690be5b3..ed8061b64f 100644 --- a/ant-node-rpc-client/Cargo.toml +++ b/ant-node-rpc-client/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-node-rpc-client" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.6.41" +version = "0.6.42" [[bin]] name = "antnode_rpc_client" @@ -17,17 +17,17 @@ path = "src/main.rs" nightly = [] [dependencies] -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3", features=["rpc"] } -ant-node = { path = "../ant-node", version = "0.3.5" } -ant-service-management = { path = "../ant-service-management", version = "0.4.7" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0", features=["rpc"] } +ant-node = { path = "../ant-node", version = "0.3.6" } +ant-service-management = { path = "../ant-service-management", version = "0.4.8" } async-trait = "0.1" bls = { package = "blsttc", version = "8.0.1" } clap = { version = "4.2.1", features = ["derive"] } -color-eyre = "0.6.2" +color-eyre = "0.6.3" hex = "~0.4.3" -libp2p = { version = "0.54.1", features = ["kad"]} +libp2p = { version = "0.55.0", features = ["kad"]} libp2p-identity = { version="0.2.7", features = ["rand"] } thiserror = "1.0.23" # # watch out updating this, protoc compiler needs to be installed on all build systems diff --git a/ant-node/Cargo.toml b/ant-node/Cargo.toml index f226244287..358c639054 100644 --- a/ant-node/Cargo.toml +++ b/ant-node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "The Autonomi node binary" name = "ant-node" -version = "0.3.5" +version = "0.3.6" edition = "2021" license = "GPL-3.0" homepage = "https://maidsafe.net" @@ -14,33 +14,28 @@ name = "antnode" path = "src/bin/antnode/main.rs" [features] -default = ["metrics", "upnp", "open-metrics", "encrypt-records"] -encrypt-records = ["ant-networking/encrypt-records"] +default = ["open-metrics"] extension-module = ["pyo3/extension-module"] -local = ["ant-networking/local", "ant-evm/local", "ant-bootstrap/local", "ant-logging/process-metrics"] loud = ["ant-networking/loud"] # loud mode: print important messages to console -metrics = [] nightly = [] open-metrics = ["ant-networking/open-metrics", "prometheus-client"] otlp = ["ant-logging/otlp"] -upnp = ["ant-networking/upnp"] [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-networking = { path = "../ant-networking", version = "0.3.4" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } -ant-registers = { path = "../ant-registers", version = "0.4.7" } -ant-service-management = { path = "../ant-service-management", version = "0.4.7" } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-logging = { path = "../ant-logging", version = "0.2.46", features = ["process-metrics"] } +ant-networking = { path = "../ant-networking", version = "0.3.5" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } +ant-service-management = { path = "../ant-service-management", version = "0.4.8" } async-trait = "0.1" bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } clap = { version = "4.2.1", features = ["derive"] } crdts = { version = "7.3", default-features = false, features = ["merkle"] } chrono = "~0.4.19" -color-eyre = "0.6.2" +color-eyre = "0.6.3" const-hex = "1.12.0" custom_debug = "~0.6.1" dirs-next = "~2.0.0" @@ -49,13 +44,13 @@ file-rotate = "0.7.3" futures = "~0.3.13" hex = "~0.4.3" itertools = "~0.12.1" -libp2p = { version = "0.54.1", features = ["tokio", "dns", "kad", "macros"] } +libp2p = { version = "0.55.0", features = ["tokio", "dns", "kad", "macros"] } num-traits = "0.2" prometheus-client = { version = "0.22", optional = true } # watch out updating this, protoc compiler needs to be installed on all build systems # arm builds + musl are very problematic prost = { version = "0.9" } -pyo3 = { version = "0.20", features = ["extension-module"], optional = true } +pyo3 = { version = "0.23.4", features = ["extension-module"], optional = true } rand = { version = "~0.8.5", features = ["small_rng"] } rmp-serde = "1.1.1" rayon = "1.8.0" @@ -77,15 +72,15 @@ tonic = { version = "0.6.2" } tracing = { version = "~0.1.26" } tracing-appender = "~0.2.0" tracing-opentelemetry = { version = "0.21", optional = true } -tracing-subscriber = { version = "0.3.16" } +tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } walkdir = "~2.5.0" xor_name = "5.0.0" [dev-dependencies] -ant-protocol = { path = "../ant-protocol", version = "0.3.3", features = ["rpc"] } +ant-protocol = { path = "../ant-protocol", version = "1.0.0", features = ["rpc"] } assert_fs = "1.0.0" -evmlib = { path = "../evmlib", version = "0.1.8" } -autonomi = { path = "../autonomi", version = "0.3.5", features = ["registers"] } +evmlib = { path = "../evmlib", version = "0.1.9" } +autonomi = { path = "../autonomi", version = "0.3.6" } reqwest = { version = "0.12.2", default-features = false, features = [ "rustls-tls-manual-roots", ] } diff --git a/ant-node/README.md b/ant-node/README.md index e95385f2e8..4ce708bfaf 100644 --- a/ant-node/README.md +++ b/ant-node/README.md @@ -120,7 +120,6 @@ default_dir = AntNode.get_default_root_dir(peer_id) - `get_validation.rs`: Validation for GET requests - `put_validation.rs`: Validation for PUT requests - `replication.rs`: Data replication logic - - `transactions.rs`: Logic related to spending tokens or resources - `tests/`: Test files - `common/mod.rs`: Common utilities for tests - `data_with_churn.rs`: Tests related to data with churn diff --git a/ant-node/examples/spawn_local_network.rs b/ant-node/examples/spawn_local_network.rs new file mode 100644 index 0000000000..782e8c3806 --- /dev/null +++ b/ant-node/examples/spawn_local_network.rs @@ -0,0 +1,41 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_node::spawn::network_spawner::NetworkSpawner; +use std::time::Duration; +use tokio::time::sleep; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{fmt, EnvFilter}; + +#[tokio::main] +async fn main() { + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_env("RUST_LOG")) + .init(); + + let network_size = 20; + + let running_network = NetworkSpawner::new() + .with_evm_network(Default::default()) + .with_local(true) + .with_size(network_size) + .spawn() + .await + .expect("Failed to spawn network"); + + // Wait for nodes to dial each other + sleep(Duration::from_secs(10)).await; + + for node in running_network.running_nodes() { + println!("Node listening on: {:?}", node.get_listen_addrs().await); + } + + running_network.shutdown(); +} diff --git a/ant-node/pyproject.toml b/ant-node/pyproject.toml index 8eda49b80d..77a5ab38a2 100644 --- a/ant-node/pyproject.toml +++ b/ant-node/pyproject.toml @@ -18,4 +18,4 @@ module-name = "antnode._antnode" python-source = "python" bindings = "pyo3" manifest-path = "Cargo.toml" -sdist-include = ["python/antnode/*"] +sdist-include = ["python/antnode/*"] \ No newline at end of file diff --git a/ant-node/python/antnode/README.md b/ant-node/python/antnode/README.md new file mode 100644 index 0000000000..91cace7bc0 --- /dev/null +++ b/ant-node/python/antnode/README.md @@ -0,0 +1,122 @@ +# AntNode Python Bindings + +This document describes the Python bindings for the AntNode Rust implementation. + +## Installation + +The AntNode Python package is built using [maturin](https://github.com/PyO3/maturin) and requires Python 3.8 or later. We recommend using `uv` for Python environment management: + +```bash +uv venv +uv pip install maturin +maturin develop +``` + +## Usage + +```python +from antnode import AntNode + +# Create a new node instance +node = AntNode() + +# Start the node with configuration +node.run( + rewards_address="0x1234567890123456789012345678901234567890", + evm_network="arbitrum_sepolia", # or "arbitrum_one" + ip="0.0.0.0", + port=12000, + initial_peers=[], # List of multiaddresses for initial peers + local=True, # Run in local mode + root_dir=None, # Custom root directory (optional) + home_network=False # Run on home network +) +``` + +## API Reference + +### Constructor + +#### `AntNode()` +Creates a new instance of the AntNode. + +### Node Operations + +#### `run(rewards_address: str, evm_network: str, ip: str = "0.0.0.0", port: int = 0, initial_peers: List[str] = [], local: bool = False, root_dir: Optional[str] = None, home_network: bool = False) -> None` +Start the node with the given configuration. + +- **Parameters:** + - `rewards_address`: Ethereum address for rewards (hex string starting with "0x") + - `evm_network`: Either "arbitrum_one" or "arbitrum_sepolia" + - `ip`: IP address to bind to (default: "0.0.0.0") + - `port`: Port number to use (default: 0 for random port) + - `initial_peers`: List of multiaddresses for initial peers + - `local`: Run in local mode + - `root_dir`: Custom root directory path (optional) + - `home_network`: Run on home network + +#### `peer_id() -> str` +Get the node's PeerId as a string. + +#### `get_rewards_address() -> str` +Get the node's rewards/wallet address as a hex string. + +#### `set_rewards_address(address: str) -> None` +Set a new rewards/wallet address for the node. +- `address`: Hex string starting with "0x" + +### Storage Operations + +#### `store_record(key: str, value: bytes, record_type: str) -> None` +Store a record in the node's storage. +- `key`: Record key +- `value`: Record data as bytes +- `record_type`: Type of record + +#### `get_record(key: str) -> Optional[bytes]` +Get a record from the node's storage. +- Returns `None` if record not found + +#### `delete_record(key: str) -> bool` +Delete a record from the node's storage. +- Returns `True` if record was deleted + +#### `get_stored_records_size() -> int` +Get the total size of stored records in bytes. + +#### `get_all_record_addresses() -> List[str]` +Get all record addresses stored by the node. + +### Network Operations + +#### `get_kbuckets() -> List[Tuple[int, List[str]]]` +Get the node's kbuckets information. +- Returns list of tuples containing (distance, list of peer IDs) + +### Directory Management + +#### `get_root_dir() -> str` +Get the current root directory path for node data. + +#### `get_default_root_dir(peer_id: Optional[str] = None) -> str` +Get the default root directory path for the given peer ID. +- Platform specific paths: + - Linux: `$HOME/.local/share/autonomi/node/` + - macOS: `$HOME/Library/Application Support/autonomi/node/` + - Windows: `C:\Users\\AppData\Roaming\autonomi\node\` + +#### `get_logs_dir() -> str` +Get the logs directory path. + +#### `get_data_dir() -> str` +Get the data directory path where records are stored. + +## Error Handling + +The bindings use Python exceptions to handle errors: +- `ValueError`: For invalid input parameters +- `RuntimeError`: For operational errors + +## Example + +See [example.py](../example.py) for a complete example of using the AntNode Python bindings. diff --git a/ant-node/python/antnode/__init__.py b/ant-node/python/antnode/__init__.py new file mode 100644 index 0000000000..949716f09f --- /dev/null +++ b/ant-node/python/antnode/__init__.py @@ -0,0 +1,21 @@ +"""AntNode Python Bindings + +This module provides Python bindings for the AntNode Rust implementation, +allowing you to run and manage AntNode instances from Python code. + +For detailed documentation, see the README.md file in this directory. + +Example: + >>> from antnode import AntNode + >>> node = AntNode() + >>> node.run( + ... rewards_address="0x1234567890123456789012345678901234567890", + ... evm_network="arbitrum_sepolia", + ... ip="0.0.0.0", + ... port=12000 + ... ) +""" + +from ._antnode import AntNode + +__all__ = ["AntNode"] diff --git a/ant-node/reactivate_examples/register_inspect.rs b/ant-node/reactivate_examples/register_inspect.rs deleted file mode 100644 index c24a87ebfa..0000000000 --- a/ant-node/reactivate_examples/register_inspect.rs +++ /dev/null @@ -1,233 +0,0 @@ -// // Copyright 2024 MaidSafe.net limited. -// // -// // This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// // Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// // under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// // KIND, either express or implied. Please review the Licences for the specific language governing -// // permissions and limitations relating to use of the SAFE Network Software. - -// use crdts::merkle_reg::{Hash, MerkleReg, Node}; -// use std::collections::HashMap; -// use std::io; - -// // TODO: use autonomi API here -// // use sn_client::{acc_packet::load_account_wallet_or_create_with_mnemonic, Client, WalletClient}; -// use ant_registers::{Entry, Permissions, RegisterAddress}; - -// use xor_name::XorName; - -// use bls::SecretKey; -// use clap::Parser; -// use color_eyre::{ -// eyre::{eyre, Result, WrapErr}, -// Help, -// }; - -// #[derive(Parser, Debug)] -// #[clap(name = "register inspect cli")] -// struct Opt { -// // Create register and give it a nickname (first user) -// #[clap(long, default_value = "")] -// reg_nickname: String, - -// // Get existing register with given network address (any other user) -// #[clap(long, default_value = "", conflicts_with = "reg_nickname")] -// reg_address: String, -// } - -// #[tokio::main] -// async fn main() -> Result<()> { -// let opt = Opt::parse(); -// let mut reg_nickname = opt.reg_nickname; -// let reg_address_string = opt.reg_address; - -// // let's build a random secret key to sign our Register ops -// let signer = SecretKey::random(); - -// println!("Starting SAFE client..."); -// let client = Client::new(signer, None, None, None).await?; -// println!("SAFE client signer public key: {:?}", client.signer_pk()); - -// // The address of the register to be displayed -// let mut meta = XorName::from_content(reg_nickname.as_bytes()); -// let reg_address = if !reg_nickname.is_empty() { -// meta = XorName::from_content(reg_nickname.as_bytes()); -// RegisterAddress::new(meta, client.signer_pk()) -// } else { -// reg_nickname = format!("{reg_address_string:<6}..."); -// RegisterAddress::from_hex(®_address_string) -// .wrap_err("cannot parse hex register address")? -// }; - -// // Loading a local wallet (for ClientRegister::sync()). -// // The wallet can have ZERO balance in this example, -// // but the ClientRegister::sync() API requires a wallet and will -// // create the register if not found even though we don't want that. -// // -// // The only want to avoid unwanted creation of a Register seems to -// // be to supply an empty wallet. -// // TODO Follow the issue about this: https://github.com/maidsafe/safe_network/issues/1308 -// let root_dir = dirs_next::data_dir() -// .ok_or_else(|| eyre!("could not obtain data directory path".to_string()))? -// .join("autonomi") -// .join("client"); - -// let wallet = load_account_wallet_or_create_with_mnemonic(&root_dir, None) -// .wrap_err(format!"Unable to read wallet file in {root_dir:?}")) -// .suggestion( -// "If you have an old wallet file, it may no longer be compatible. Try removing it", -// )?; - -// let mut wallet_client = WalletClient::new(client.clone(), wallet); - -// println!("Retrieving Register '{reg_nickname}' from SAFE"); -// let mut reg_replica = match client.get_register(reg_address).await { -// Ok(register) => { -// println!( -// "Register '{reg_nickname}' found at {:?}!", -// register.address(), -// ); -// register -// } -// Err(_) => { -// println!("Register '{reg_nickname}' not found, creating it at {reg_address}"); -// let (register, _cost, _royalties_fees) = client -// .create_and_pay_for_register( -// meta, -// &mut wallet_client, -// true, -// Permissions::new_anyone_can_write(), -// ) -// .await?; - -// register -// } -// }; -// println!("Register address: {:?}", reg_replica.address().to_hex()); -// println!("Register owned by: {:?}", reg_replica.owner()); -// println!("Register permissions: {:?}", reg_replica.permissions()); - -// // Repeatedly display of the register structure on command -// loop { -// println!(); -// println!( -// "Current total number of items in Register: {}", -// reg_replica.size() -// ); -// println!("Latest value (more than one if concurrent writes were made):"); -// println!("--------------"); -// for (_, entry) in reg_replica.read().into_iter() { -// println!("{}", String::from_utf8(entry)?); -// } -// println!("--------------"); - -// if prompt_user() { -// return Ok(()); -// } - -// // Sync with network after a delay -// println!("Syncing with SAFE..."); -// reg_replica.sync(&mut wallet_client, true, None).await?; -// let merkle_reg = reg_replica.merkle_reg(); -// let content = merkle_reg.read(); -// println!("synced!"); - -// // Show the Register structure - -// // Index nodes to make it easier to see where a -// // node appears multiple times in the output. -// // Note: it isn't related to the order of insertion -// // which is hard to determine. -// let mut index: usize = 0; -// let mut node_ordering: HashMap = HashMap::new(); -// for (_hash, node) in content.hashes_and_nodes() { -// index_node_and_descendants(node, &mut index, &mut node_ordering, merkle_reg); -// } - -// println!("======================"); -// println!("Root (Latest) Node(s):"); -// for node in content.nodes() { -// let _ = print_node(0, node, &node_ordering); -// } - -// println!("======================"); -// println!("Register Structure:"); -// println!("(In general, earlier nodes are more indented)"); -// let mut indents = 0; -// for (_hash, node) in content.hashes_and_nodes() { -// print_node_and_descendants(&mut indents, node, &node_ordering, merkle_reg); -// } - -// println!("======================"); -// } -// } - -// fn index_node_and_descendants( -// node: &Node, -// index: &mut usize, -// node_ordering: &mut HashMap, -// merkle_reg: &MerkleReg, -// ) { -// let node_hash = node.hash(); -// if node_ordering.get(&node_hash).is_none() { -// node_ordering.insert(node_hash, *index); -// *index += 1; -// } - -// for child_hash in node.children.iter() { -// if let Some(child_node) = merkle_reg.node(*child_hash) { -// index_node_and_descendants(child_node, index, node_ordering, merkle_reg); -// } else { -// println!("ERROR looking up hash of child"); -// } -// } -// } - -// fn print_node_and_descendants( -// indents: &mut usize, -// node: &Node, -// node_ordering: &HashMap, -// merkle_reg: &MerkleReg, -// ) { -// let _ = print_node(*indents, node, node_ordering); - -// *indents += 1; -// for child_hash in node.children.iter() { -// if let Some(child_node) = merkle_reg.node(*child_hash) { -// print_node_and_descendants(indents, child_node, node_ordering, merkle_reg); -// } -// } -// *indents -= 1; -// } - -// fn print_node( -// indents: usize, -// node: &Node, -// node_ordering: &HashMap, -// ) -> Result<()> { -// let order = match node_ordering.get(&node.hash()) { -// Some(order) => format!("{order}"), -// None => String::new(), -// }; -// let indentation = " ".repeat(indents); -// println!( -// "{indentation}[{:>2}] Node({:?}..) Entry({:?})", -// order, -// hex::encode(&node.hash()[0..3]), -// String::from_utf8(node.value.clone())? -// ); -// Ok(()) -// } - -// fn prompt_user() -> bool { -// let mut input_text = String::new(); -// println!(); -// println!("Enter a blank line to print the latest register structure (or 'Q' to quit)"); -// io::stdin() -// .read_line(&mut input_text) -// .expect("Failed to read text from stdin"); - -// let string = input_text.trim().to_string(); - -// string.contains('Q') || string.contains('q') -// } diff --git a/ant-node/reactivate_examples/registers.rs b/ant-node/reactivate_examples/registers.rs deleted file mode 100644 index 4f9f7e5fcb..0000000000 --- a/ant-node/reactivate_examples/registers.rs +++ /dev/null @@ -1,167 +0,0 @@ -// // Copyright 2024 MaidSafe.net limited. -// // -// // This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// // Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// // under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// // KIND, either express or implied. Please review the Licences for the specific language governing -// // permissions and limitations relating to use of the SAFE Network Software. - -// // TODO: use autonomi API here. -// // use sn_client::{ -// // acc_packet::load_account_wallet_or_create_with_mnemonic, Client, Error, WalletClient, -// // }; -// use ant_registers::{Permissions, RegisterAddress}; - -// use xor_name::XorName; - -// use bls::SecretKey; -// use clap::Parser; -// use color_eyre::{ -// eyre::{eyre, Result, WrapErr}, -// Help, -// }; -// use std::{io, time::Duration}; -// use tokio::time::sleep; - -// #[derive(Parser, Debug)] -// #[clap(name = "registers cli")] -// struct Opt { -// // A name for this user in the example -// #[clap(long)] -// user: String, - -// // Create register and give it a nickname (first user) -// #[clap(long, default_value = "")] -// reg_nickname: String, - -// // Get existing register with given network address (any other user) -// #[clap(long, default_value = "", conflicts_with = "reg_nickname")] -// reg_address: String, - -// // Delay before synchronising local register with the network -// #[clap(long, default_value_t = 2000)] -// delay_millis: u64, -// } - -// #[tokio::main] -// async fn main() -> Result<()> { -// let opt = Opt::parse(); -// let user = opt.user; -// let mut reg_nickname = opt.reg_nickname; -// let reg_address_string = opt.reg_address; -// let delay = Duration::from_millis(opt.delay_millis); - -// // let's build a random secret key to sign our Register ops -// let signer = SecretKey::random(); - -// println!("Starting SAFE client..."); -// let client = Client::new(signer, None, None, None).await?; -// println!("SAFE client signer public key: {:?}", client.signer_pk()); - -// // We'll retrieve (or create if not found) a Register, and write on it -// // in offline mode, syncing with the network periodically. - -// let mut meta = XorName::from_content(reg_nickname.as_bytes()); -// let reg_address = if !reg_nickname.is_empty() { -// meta = XorName::from_content(reg_nickname.as_bytes()); -// RegisterAddress::new(meta, client.signer_pk()) -// } else { -// reg_nickname = format!("{reg_address_string:<6}..."); -// RegisterAddress::from_hex(®_address_string) -// .wrap_err("cannot parse hex register address")? -// }; - -// // Loading a local wallet. It needs to have a non-zero balance for -// // this example to be able to pay for the Register's storage. -// let root_dir = dirs_next::data_dir() -// .ok_or_else(|| eyre!("could not obtain data directory path".to_string()))? -// .join("autonomi") -// .join("client"); - -// let wallet = load_account_wallet_or_create_with_mnemonic(&root_dir, None) -// .wrap_err("Unable to read wallet file in {root_dir:?}") -// .suggestion( -// "If you have an old wallet file, it may no longer be compatible. Try removing it", -// )?; -// let mut wallet_client = WalletClient::new(client.clone(), wallet); - -// println!("Retrieving Register '{reg_nickname}' from SAFE, as user '{user}'"); -// let mut reg_replica = match client.get_register(reg_address).await { -// Ok(register) => { -// println!( -// "Register '{reg_nickname}' found at {:?}!", -// register.address(), -// ); -// register -// } -// Err(_) => { -// println!("Register '{reg_nickname}' not found, creating it at {reg_address}"); -// let (register, _cost, _royalties_fees) = client -// .create_and_pay_for_register( -// meta, -// &mut wallet_client, -// true, -// Permissions::new_anyone_can_write(), -// ) -// .await?; - -// register -// } -// }; -// println!("Register address: {:?}", reg_replica.address().to_hex()); -// println!("Register owned by: {:?}", reg_replica.owner()); -// println!("Register permissions: {:?}", reg_replica.permissions()); - -// // We'll loop asking for new msg to write onto the Register offline, -// // then we'll be syncing the offline Register with the network, i.e. -// // both pushing and ulling all changes made to it by us and other clients/users. -// // If we detect branches when trying to write, after we synced with remote -// // replicas of the Register, we'll merge them all back into a single value. -// loop { -// println!(); -// println!( -// "Current total number of items in Register: {}", -// reg_replica.size() -// ); -// println!("Latest value (more than one if concurrent writes were made):"); -// println!("--------------"); -// for (_, entry) in reg_replica.read().into_iter() { -// println!("{}", String::from_utf8(entry)?); -// } -// println!("--------------"); - -// let input_text = prompt_user(); -// if !input_text.is_empty() { -// println!("Writing msg (offline) to Register: '{input_text}'"); -// let msg = format!("[{user}]: {input_text}"); -// match reg_replica.write(msg.as_bytes()) { -// Ok(_) => {} -// Err(Error::ContentBranchDetected(branches)) => { -// println!( -// "Branches ({}) detected in Register, let's merge them all...", -// branches.len() -// ); -// reg_replica.write_merging_branches(msg.as_bytes())?; -// } -// Err(err) => return Err(err.into()), -// } -// } - -// // Sync with network after a delay -// println!("Syncing with SAFE in {delay:?}..."); -// sleep(delay).await; -// reg_replica.sync(&mut wallet_client, true, None).await?; -// println!("synced!"); -// } -// } - -// fn prompt_user() -> String { -// let mut input_text = String::new(); -// println!(); -// println!("Enter a blank line to receive updates, or some text to be written."); -// io::stdin() -// .read_line(&mut input_text) -// .expect("Failed to read text from stdin"); - -// input_text.trim().to_string() -// } diff --git a/ant-node/src/bin/antnode/main.rs b/ant-node/src/bin/antnode/main.rs index 1cd7a3e79d..5e8fbb95b8 100644 --- a/ant-node/src/bin/antnode/main.rs +++ b/ant-node/src/bin/antnode/main.rs @@ -13,11 +13,11 @@ mod rpc_service; mod subcommands; use crate::subcommands::EvmNetworkCommand; -use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore, PeersArgs}; -use ant_evm::{get_evm_network_from_env, EvmNetwork, RewardsAddress}; -#[cfg(feature = "local")] +use ant_bootstrap::{BootstrapCacheStore, PeersArgs}; +use ant_evm::{get_evm_network, EvmNetwork, RewardsAddress}; use ant_logging::metrics::init_metrics; use ant_logging::{Level, LogFormat, LogOutputDest, ReloadHandle}; +use ant_node::utils::get_root_dir_and_keypair; use ant_node::{Marker, NodeBuilder, NodeEvent, NodeEventsReceiver}; use ant_protocol::{ node::get_antnode_root_dir, @@ -27,12 +27,11 @@ use ant_protocol::{ use clap::{command, Parser}; use color_eyre::{eyre::eyre, Result}; use const_hex::traits::FromHex; -use libp2p::{identity::Keypair, PeerId}; +use libp2p::PeerId; use std::{ env, - io::Write, net::{IpAddr, Ipv4Addr, SocketAddr}, - path::{Path, PathBuf}, + path::PathBuf, process::Command, time::Duration, }; @@ -84,7 +83,6 @@ struct Opt { home_network: bool, /// Try to use UPnP to open a port in the home router and allow incoming connections. - #[cfg(feature = "upnp")] #[clap(long, default_value_t = false)] upnp: bool, @@ -177,10 +175,6 @@ struct Opt { #[clap(long)] rpc: Option, - /// Specify the owner(readable discord user name). - #[clap(long)] - owner: Option, - #[cfg(feature = "open-metrics")] /// Specify the port for the OpenMetrics server. /// @@ -262,12 +256,16 @@ fn main() -> Result<()> { return Ok(()); } - let evm_network: EvmNetwork = opt - .evm_network - .as_ref() - .cloned() - .map(|v| Ok(v.into())) - .unwrap_or_else(get_evm_network_from_env)?; + let evm_network: EvmNetwork = match opt.evm_network.as_ref() { + Some(evm_network) => Ok(evm_network.clone().into()), + None => match get_evm_network(opt.peers.local) { + Ok(net) => Ok(net), + Err(_) => Err(eyre!( + "EVM network not specified. Please specify a network using the subcommand or by setting the `EVM_NETWORK` environment variable." + )), + }, + }?; + println!("EVM network: {evm_network:?}"); let node_socket_addr = SocketAddr::new(opt.ip, opt.port); @@ -276,13 +274,14 @@ fn main() -> Result<()> { let (log_output_dest, log_reload_handle, _log_appender_guard) = init_logging(&opt, keypair.public().to_peer_id())?; - let rt = Runtime::new()?; - let mut bootstrap_cache = BootstrapCacheStore::new_from_peers_args( - &opt.peers, - Some(BootstrapCacheConfig::default_config()?), - )?; - // To create the file before startup if it doesn't exist. - bootstrap_cache.sync_and_flush_to_disk(true)?; + let mut bootstrap_cache = BootstrapCacheStore::new_from_peers_args(&opt.peers, None)?; + // If we are the first node, write initial cache to disk. + if opt.peers.first { + bootstrap_cache.write()?; + } else { + // Else we check/clean the file, write it back, and ensure its existence. + bootstrap_cache.sync_and_flush_to_disk(true)?; + } let msg = format!( "Running {} v{}", @@ -305,10 +304,11 @@ fn main() -> Result<()> { // Create a tokio runtime per `run_node` attempt, this ensures // any spawned tasks are closed before we would attempt to run // another process with these args. - #[cfg(feature = "local")] - rt.spawn(init_metrics(std::process::id())); - let initial_peres = rt.block_on(opt.peers.get_addrs(None, Some(100)))?; - debug!("Node's owner set to: {:?}", opt.owner); + let rt = Runtime::new()?; + if opt.peers.local { + rt.spawn(init_metrics(std::process::id())); + } + let initial_peers = rt.block_on(opt.peers.get_addrs(None, Some(100)))?; let restart_options = rt.block_on(async move { let mut node_builder = NodeBuilder::new( keypair, @@ -317,10 +317,9 @@ fn main() -> Result<()> { node_socket_addr, opt.peers.local, root_dir, - #[cfg(feature = "upnp")] opt.upnp, ); - node_builder.initial_peers(initial_peres); + node_builder.initial_peers(initial_peers); node_builder.bootstrap_cache(bootstrap_cache); node_builder.is_behind_home_network(opt.home_network); #[cfg(feature = "open-metrics")] @@ -521,7 +520,6 @@ fn init_logging(opt: &Opt, peer_id: PeerId) -> Result<(String, ReloadHandle, Opt ("ant_networking".to_string(), Level::INFO), ("ant_node".to_string(), Level::DEBUG), ("ant_protocol".to_string(), Level::DEBUG), - ("ant_registers".to_string(), Level::DEBUG), ("antnode".to_string(), Level::DEBUG), ]; @@ -571,81 +569,6 @@ fn init_logging(opt: &Opt, peer_id: PeerId) -> Result<(String, ReloadHandle, Opt Ok((output_dest.to_string(), reload_handle, log_appender_guard)) } -fn create_secret_key_file(path: impl AsRef) -> Result { - let mut opt = std::fs::OpenOptions::new(); - opt.write(true).create_new(true); - - // On Unix systems, make sure only the current user can read/write. - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - opt.mode(0o600); - } - - opt.open(path) -} - -fn keypair_from_path(path: impl AsRef) -> Result { - let keypair = match std::fs::read(&path) { - // If the file is opened successfully, read the key from it - Ok(key) => { - let keypair = Keypair::ed25519_from_bytes(key) - .map_err(|err| eyre!("could not read ed25519 key from file: {err}"))?; - - info!("loaded secret key from file: {:?}", path.as_ref()); - - keypair - } - // In case the file is not found, generate a new keypair and write it to the file - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - let secret_key = libp2p::identity::ed25519::SecretKey::generate(); - let mut file = create_secret_key_file(&path) - .map_err(|err| eyre!("could not create secret key file: {err}"))?; - file.write_all(secret_key.as_ref())?; - - info!("generated new key and stored to file: {:?}", path.as_ref()); - - libp2p::identity::ed25519::Keypair::from(secret_key).into() - } - // Else the file can't be opened, for whatever reason (e.g. permissions). - Err(err) => { - return Err(eyre!("failed to read secret key file: {err}")); - } - }; - - Ok(keypair) -} - -/// The keypair is located inside the root directory. At the same time, when no dir is specified, -/// the dir name is derived from the keypair used in the application: the peer ID is used as the directory name. -fn get_root_dir_and_keypair(root_dir: &Option) -> Result<(PathBuf, Keypair)> { - match root_dir { - Some(dir) => { - std::fs::create_dir_all(dir)?; - - let secret_key_path = dir.join("secret-key"); - Ok((dir.clone(), keypair_from_path(secret_key_path)?)) - } - None => { - let secret_key = libp2p::identity::ed25519::SecretKey::generate(); - let keypair: Keypair = - libp2p::identity::ed25519::Keypair::from(secret_key.clone()).into(); - let peer_id = keypair.public().to_peer_id(); - - let dir = get_antnode_root_dir(peer_id)?; - std::fs::create_dir_all(&dir)?; - - let secret_key_path = dir.join("secret-key"); - - let mut file = create_secret_key_file(secret_key_path) - .map_err(|err| eyre!("could not create secret key file: {err}"))?; - file.write_all(secret_key.as_ref())?; - - Ok((dir, keypair)) - } - } -} - /// Starts a new process running the binary with the same args as /// the current process /// Optionally provide the node's root dir and listen port to retain it's PeerId @@ -654,11 +577,14 @@ fn start_new_node_process(retain_peer_id: bool, root_dir: PathBuf, port: u16) { let current_exe = env::current_exe().expect("could not get current executable path"); // Retrieve the command-line arguments passed to this process - let args: Vec = env::args().collect(); + let mut args: Vec = env::args().collect(); info!("Original args are: {args:?}"); info!("Current exe is: {current_exe:?}"); + // Remove `--first` argument. If node is restarted, it is not the first anymore. + args.retain(|arg| arg != "--first"); + // Convert current exe path to string, log an error and return if it fails let current_exe = match current_exe.to_str() { Some(s) => { diff --git a/ant-node/src/bin/antnode/subcommands.rs b/ant-node/src/bin/antnode/subcommands.rs index a9e02d2be4..52c48f1ea7 100644 --- a/ant-node/src/bin/antnode/subcommands.rs +++ b/ant-node/src/bin/antnode/subcommands.rs @@ -10,6 +10,9 @@ pub(crate) enum EvmNetworkCommand { /// Use the Arbitrum Sepolia network EvmArbitrumSepolia, + /// Use the Arbitrum Sepolia network with test contracts + EvmArbitrumSepoliaTest, + /// Use a custom network EvmCustom { /// The RPC URL for the custom network @@ -32,6 +35,7 @@ impl Into for EvmNetworkCommand { match self { Self::EvmArbitrumOne => EvmNetwork::ArbitrumOne, Self::EvmArbitrumSepolia => EvmNetwork::ArbitrumSepolia, + Self::EvmArbitrumSepoliaTest => EvmNetwork::ArbitrumSepoliaTest, Self::EvmCustom { rpc_url, payment_token_address, diff --git a/ant-node/src/error.rs b/ant-node/src/error.rs index 6cc7f3baf1..cebe3f3ac7 100644 --- a/ant-node/src/error.rs +++ b/ant-node/src/error.rs @@ -12,6 +12,8 @@ use thiserror::Error; pub(super) type Result = std::result::Result; +const SCRATCHPAD_MAX_SIZE: usize = ant_protocol::storage::Scratchpad::MAX_SIZE; + /// Internal error. #[derive(Debug, Error)] #[allow(missing_docs)] @@ -22,9 +24,6 @@ pub enum Error { #[error("Protocol error {0}")] Protocol(#[from] ant_protocol::Error), - #[error("Register error {0}")] - Register(#[from] ant_registers::Error), - #[error("Transfers Error {0}")] Transfers(#[from] ant_evm::EvmError), @@ -41,12 +40,16 @@ pub enum Error { #[error("The Record::key does not match with the key derived from Record::value")] RecordKeyMismatch, - // Scratchpad is old version + // ------------ Scratchpad Errors #[error("A newer version of this Scratchpad already exists")] IgnoringOutdatedScratchpadPut, - // Scratchpad is invalid - #[error("Scratchpad signature is invalid over the counter + content hash")] + #[error("Scratchpad signature is invalid")] InvalidScratchpadSignature, + #[error("Scratchpad too big: {0}, max size is {SCRATCHPAD_MAX_SIZE}")] + ScratchpadTooBig(usize), + + #[error("Invalid signature")] + InvalidSignature, // ---------- Payment Errors #[error("The content of the payment quote is invalid")] diff --git a/ant-node/src/event.rs b/ant-node/src/event.rs index d8b508ec74..223764571a 100644 --- a/ant-node/src/event.rs +++ b/ant-node/src/event.rs @@ -9,10 +9,7 @@ use crate::error::{Error, Result}; use ant_evm::AttoTokens; -use ant_protocol::{ - storage::{ChunkAddress, RegisterAddress}, - NetworkAddress, -}; +use ant_protocol::{storage::ChunkAddress, NetworkAddress}; use serde::{Deserialize, Serialize}; use tokio::sync::broadcast; @@ -61,10 +58,6 @@ pub enum NodeEvent { ConnectedToNetwork, /// A Chunk has been stored in local storage ChunkStored(ChunkAddress), - /// A Register has been created in local storage - RegisterCreated(RegisterAddress), - /// A Register edit operation has been applied in local storage - RegisterEdited(RegisterAddress), /// A new reward was received RewardReceived(AttoTokens, NetworkAddress), /// One of the sub event channel closed and unrecoverable. diff --git a/ant-node/src/lib.rs b/ant-node/src/lib.rs index 3599e14a7a..558b3ecb55 100644 --- a/ant-node/src/lib.rs +++ b/ant-node/src/lib.rs @@ -32,6 +32,10 @@ mod put_validation; mod python; mod quote; mod replication; +#[allow(missing_docs)] +pub mod spawn; +#[allow(missing_docs)] +pub mod utils; pub use self::{ event::{NodeEvent, NodeEventsChannel, NodeEventsReceiver}, @@ -41,20 +45,22 @@ pub use self::{ use crate::error::{Error, Result}; +use ant_evm::RewardsAddress; use ant_networking::{Network, SwarmLocalState}; use ant_protocol::{get_port_from_multiaddr, NetworkAddress}; -use libp2p::PeerId; +use libp2p::{Multiaddr, PeerId}; + use std::{ collections::{BTreeMap, HashSet}, path::PathBuf, }; - -use ant_evm::RewardsAddress; +use tokio::sync::watch; /// Once a node is started and running, the user obtains /// a `NodeRunning` object which can be used to interact with it. #[derive(Clone)] pub struct RunningNode { + shutdown_sender: watch::Sender, network: Network, node_events_channel: NodeEventsChannel, root_dir_path: PathBuf, @@ -85,6 +91,24 @@ impl RunningNode { Ok(state) } + /// Return the node's listening addresses. + pub async fn get_listen_addrs(&self) -> Result> { + let listeners = self.network.get_swarm_local_state().await?.listeners; + Ok(listeners) + } + + /// Return the node's listening addresses with the peer id appended. + pub async fn get_listen_addrs_with_peer_id(&self) -> Result> { + let listeners = self.get_listen_addrs().await?; + + let multi_addrs: Vec = listeners + .into_iter() + .filter_map(|listen_addr| listen_addr.with_p2p(self.peer_id()).ok()) + .collect(); + + Ok(multi_addrs) + } + /// Return the node's listening port pub async fn get_node_listening_port(&self) -> Result { let listen_addrs = self.network.get_swarm_local_state().await?.listeners; @@ -125,4 +149,10 @@ impl RunningNode { pub fn reward_address(&self) -> &RewardsAddress { &self.rewards_address } + + /// Shutdown the SwarmDriver loop and the node (NetworkEvents) loop. + pub fn shutdown(self) { + // Send the shutdown signal to the swarm driver and node loop + let _ = self.shutdown_sender.send(true); + } } diff --git a/ant-node/src/log_markers.rs b/ant-node/src/log_markers.rs index 23f7c0829e..3e833dfd5a 100644 --- a/ant-node/src/log_markers.rs +++ b/ant-node/src/log_markers.rs @@ -39,22 +39,21 @@ pub enum Marker<'a> { /// Valid non-existing Chunk record PUT from the network received and stored ValidChunkRecordPutFromNetwork(&'a PrettyPrintRecordKey<'a>), - /// Valid non-existing Register record PUT from the network received and stored - ValidRegisterRecordPutFromNetwork(&'a PrettyPrintRecordKey<'a>), - /// Valid non-existing Spend record PUT from the network received and stored - ValidTransactionRecordPutFromNetwork(&'a PrettyPrintRecordKey<'a>), + /// Valid non-existing GraphEntry record PUT from the network received and stored + ValidGraphEntryRecordPutFromNetwork(&'a PrettyPrintRecordKey<'a>), /// Valid Scratchpad record PUT from the network received and stored ValidScratchpadRecordPutFromNetwork(&'a PrettyPrintRecordKey<'a>), /// Valid paid to us and royalty paid chunk stored ValidPaidChunkPutFromClient(&'a PrettyPrintRecordKey<'a>), - /// Valid paid to us and royalty paid register stored - ValidPaidRegisterPutFromClient(&'a PrettyPrintRecordKey<'a>), - /// Valid transaction stored - ValidTransactionPutFromClient(&'a PrettyPrintRecordKey<'a>), + /// Valid GraphEntry stored + ValidGraphEntryPutFromClient(&'a PrettyPrintRecordKey<'a>), /// Valid scratchpad stored ValidScratchpadRecordPutFromClient(&'a PrettyPrintRecordKey<'a>), + /// Valid paid to us and royalty paid pointer stored + ValidPointerPutFromClient(&'a PrettyPrintRecordKey<'a>), + /// Record rejected RecordRejected(&'a PrettyPrintRecordKey<'a>, &'a Error), diff --git a/ant-node/src/metrics.rs b/ant-node/src/metrics.rs index 43bad46639..0ad237b6ee 100644 --- a/ant-node/src/metrics.rs +++ b/ant-node/src/metrics.rs @@ -7,11 +7,12 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::Marker; -use ant_networking::target_arch::Instant; +use ant_networking::time::Instant; #[cfg(feature = "open-metrics")] use ant_networking::MetricsRegistries; +use ant_protocol::storage::DataTypes; use prometheus_client::{ - encoding::{EncodeLabelSet, EncodeLabelValue}, + encoding::EncodeLabelSet, metrics::{ counter::Counter, family::Family, @@ -47,14 +48,7 @@ pub(crate) struct NodeMetricsRecorder { #[derive(EncodeLabelSet, Hash, Clone, Eq, PartialEq, Debug)] struct PutRecordOk { - record_type: RecordType, -} - -#[derive(EncodeLabelValue, Hash, Clone, Eq, PartialEq, Debug)] -enum RecordType { - Chunk, - Register, - Spend, + record_type: DataTypes, } impl NodeMetricsRecorder { @@ -157,25 +151,16 @@ impl NodeMetricsRecorder { let _ = self .put_record_ok .get_or_create(&PutRecordOk { - record_type: RecordType::Chunk, - }) - .inc(); - } - - Marker::ValidRegisterRecordPutFromNetwork(_) => { - let _ = self - .put_record_ok - .get_or_create(&PutRecordOk { - record_type: RecordType::Register, + record_type: DataTypes::Chunk, }) .inc(); } - Marker::ValidTransactionRecordPutFromNetwork(_) => { + Marker::ValidGraphEntryRecordPutFromNetwork(_) => { let _ = self .put_record_ok .get_or_create(&PutRecordOk { - record_type: RecordType::Spend, + record_type: DataTypes::GraphEntry, }) .inc(); } diff --git a/ant-node/src/node.rs b/ant-node/src/node.rs index 2515af6344..86ad2f47c4 100644 --- a/ant-node/src/node.rs +++ b/ant-node/src/node.rs @@ -13,22 +13,22 @@ use super::{ use crate::metrics::NodeMetricsRecorder; use crate::RunningNode; use ant_bootstrap::BootstrapCacheStore; +use ant_evm::EvmNetwork; use ant_evm::RewardsAddress; #[cfg(feature = "open-metrics")] use ant_networking::MetricsRegistries; use ant_networking::{ - target_arch::sleep, Instant, Network, NetworkBuilder, NetworkEvent, NodeIssue, SwarmDriver, + time::sleep, Instant, Network, NetworkBuilder, NetworkEvent, NodeIssue, SwarmDriver, }; use ant_protocol::{ - convert_distance_to_u256, error::Error as ProtocolError, messages::{ChunkProof, CmdResponse, Nonce, Query, QueryResponse, Request, Response}, - storage::RecordType, + storage::ValidationType, NetworkAddress, PrettyPrintRecordKey, CLOSE_GROUP_SIZE, }; use bytes::Bytes; use itertools::Itertools; -use libp2p::{identity::Keypair, Multiaddr, PeerId}; +use libp2p::{identity::Keypair, kad::U256, Multiaddr, PeerId}; use num_traits::cast::ToPrimitive; use rand::{ rngs::{OsRng, StdRng}, @@ -44,13 +44,12 @@ use std::{ }, time::Duration, }; +use tokio::sync::watch; use tokio::{ sync::mpsc::Receiver, task::{spawn, JoinSet}, }; -use ant_evm::{EvmNetwork, U256}; - /// Interval to trigger replication of all records to all peers. /// This is the max time it should take. Minimum interval at any node will be half this pub const PERIODIC_REPLICATION_INTERVAL_MAX_S: u64 = 180; @@ -70,7 +69,7 @@ const HIGHEST_SCORE: usize = 100; /// Any nodes bearing a score below this shall be considered as bad. /// Max is to be 100 * 100 -const MIN_ACCEPTABLE_HEALTHY_SCORE: usize = 5000; +const MIN_ACCEPTABLE_HEALTHY_SCORE: usize = 3000; /// in ms, expecting average StorageChallenge complete time to be around 250ms. const TIME_STEP: usize = 20; @@ -94,7 +93,6 @@ pub struct NodeBuilder { metrics_server_port: Option, /// Enable hole punching for nodes connecting from home networks. is_behind_home_network: bool, - #[cfg(feature = "upnp")] upnp: bool, } @@ -108,7 +106,7 @@ impl NodeBuilder { addr: SocketAddr, local: bool, root_dir: PathBuf, - #[cfg(feature = "upnp")] upnp: bool, + upnp: bool, ) -> Self { Self { bootstrap_cache: None, @@ -122,7 +120,6 @@ impl NodeBuilder { #[cfg(feature = "open-metrics")] metrics_server_port: None, is_behind_home_network: false, - #[cfg(feature = "upnp")] upnp, } } @@ -184,11 +181,11 @@ impl NodeBuilder { network_builder.bootstrap_cache(cache); } - #[cfg(feature = "upnp")] network_builder.upnp(self.upnp); let (network, network_event_receiver, swarm_driver) = network_builder.build_node(self.root_dir.clone())?; + let node_events_channel = NodeEventsChannel::default(); let node = NodeInner { @@ -200,19 +197,25 @@ impl NodeBuilder { metrics_recorder, evm_network: self.evm_network, }; + let node = Node { inner: Arc::new(node), }; + + // Create a shutdown signal channel + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + // Run the node + node.run(swarm_driver, network_event_receiver, shutdown_rx); + let running_node = RunningNode { + shutdown_sender: shutdown_tx, network, node_events_channel, root_dir_path: self.root_dir, rewards_address: self.evm_address, }; - // Run the node - node.run(swarm_driver, network_event_receiver); - Ok(running_node) } } @@ -270,14 +273,20 @@ impl Node { &self.inner.evm_network } - /// Runs the provided `SwarmDriver` and spawns a task to process for `NetworkEvents` - fn run(self, swarm_driver: SwarmDriver, mut network_event_receiver: Receiver) { + /// Runs a task for the provided `SwarmDriver` and spawns a task to process for `NetworkEvents`. + /// Returns both tasks as JoinHandle<()>. + fn run( + self, + swarm_driver: SwarmDriver, + mut network_event_receiver: Receiver, + mut shutdown_rx: watch::Receiver, + ) { let mut rng = StdRng::from_entropy(); let peers_connected = Arc::new(AtomicUsize::new(0)); - let _handle = spawn(swarm_driver.run()); - let _handle = spawn(async move { + let _swarm_driver_task = spawn(swarm_driver.run(shutdown_rx.clone())); + let _node_task = spawn(async move { // use a random inactivity timeout to ensure that the nodes do not sync when messages // are being transmitted. let replication_interval: u64 = rng.gen_range( @@ -328,6 +337,13 @@ impl Node { let peers_connected = &peers_connected; tokio::select! { + // Check for a shutdown command. + result = shutdown_rx.changed() => { + if result.is_ok() && *shutdown_rx.borrow() || result.is_err() { + info!("Shutdown signal received or sender dropped. Exiting network events loop."); + break; + } + }, net_event = network_event_receiver.recv() => { match net_event { Some(event) => { @@ -348,7 +364,6 @@ impl Node { // runs every replication_interval time _ = replication_interval.tick() => { let start = Instant::now(); - debug!("Periodic replication triggered"); let network = self.network().clone(); self.record_metrics(Marker::IntervalReplicationTriggered); @@ -456,28 +471,24 @@ impl Node { } NetworkEvent::NewListenAddr(_) => { event_header = "NewListenAddr"; - if !cfg!(feature = "local") { - let network = self.network().clone(); - let peers = self.initial_peers().clone(); - let _handle = spawn(async move { - for addr in peers { - if let Err(err) = network.dial(addr.clone()).await { - tracing::error!("Failed to dial {addr}: {err:?}"); - }; - } - }); - } + let network = self.network().clone(); + let peers = self.initial_peers().clone(); + let _handle = spawn(async move { + for addr in peers { + if let Err(err) = network.dial(addr.clone()).await { + tracing::error!("Failed to dial {addr}: {err:?}"); + }; + } + }); } NetworkEvent::ResponseReceived { res } => { event_header = "ResponseReceived"; - debug!("NetworkEvent::ResponseReceived {res:?}"); if let Err(err) = self.handle_response(res) { error!("Error while handling NetworkEvent::ResponseReceived {err:?}"); } } NetworkEvent::KeysToFetchForReplication(keys) => { event_header = "KeysToFetchForReplication"; - debug!("Going to fetch {:?} keys for replication", keys.len()); self.record_metrics(Marker::fetching_keys_for_replication(&keys)); if let Err(err) = self.fetch_replication_keys_without_wait(keys) { @@ -523,10 +534,19 @@ impl Node { // Note: this log will be checked in CI, and expecting `not appear`. // any change to the keyword `failed to fetch` shall incur // correspondent CI script change as well. - error!("Received notification from replication_fetcher, notifying {bad_nodes:?} failed to fetch replication copies from."); + debug!("Received notification from replication_fetcher, notifying {bad_nodes:?} failed to fetch replication copies from."); let _handle = spawn(async move { - for peer_id in bad_nodes { - network.record_node_issues(peer_id, NodeIssue::ReplicationFailure); + for (peer_id, record_key) in bad_nodes { + // Obsoleted fetch request (due to flooded in fresh replicates) could result + // in peer to be claimed as bad, as local copy blocks the entry to be cleared. + if let Ok(false) = network.is_record_key_present_locally(&record_key).await + { + error!( + "From peer {peer_id:?}, failed to fetch record {:?}", + PrettyPrintRecordKey::from(&record_key) + ); + network.record_node_issues(peer_id, NodeIssue::ReplicationFailure); + } } }); } @@ -538,6 +558,10 @@ impl Node { quotes_verification(&network, quotes).await; }); } + NetworkEvent::FreshReplicateToFetch { holder, keys } => { + event_header = "FreshReplicateToFetch"; + self.fresh_replicate_to_fetch(holder, keys); + } } trace!( @@ -556,6 +580,9 @@ impl Node { Response::Query(QueryResponse::GetReplicatedRecord(resp)) => { error!("Response to replication shall be handled by called not by common handler, {resp:?}"); } + Response::Cmd(CmdResponse::FreshReplicate(Ok(()))) => { + // No need to handle + } other => { warn!("handle_response not implemented for {other:?}"); } @@ -572,15 +599,17 @@ impl Node { let resp: QueryResponse = match query { Query::GetStoreQuote { key, + data_type, + data_size, nonce, difficulty, } => { - debug!("Got GetStoreQuote request for {key:?} with difficulty {difficulty}"); let record_key = key.to_record_key(); let self_id = network.peer_id(); - let maybe_quoting_metrics = - network.get_local_quoting_metrics(record_key.clone()).await; + let maybe_quoting_metrics = network + .get_local_quoting_metrics(record_key.clone(), data_type, data_size) + .await; let storage_proofs = if let Some(nonce) = nonce { Self::respond_x_closest_record_proof( @@ -628,27 +657,7 @@ impl Node { } } } - Query::GetRegisterRecord { requester, key } => { - debug!("Got GetRegisterRecord from {requester:?} regarding {key:?} "); - - let our_address = NetworkAddress::from_peer(network.peer_id()); - let mut result = Err(ProtocolError::RegisterRecordNotFound { - holder: Box::new(our_address.clone()), - key: Box::new(key.clone()), - }); - let record_key = key.as_record_key(); - - if let Some(record_key) = record_key { - if let Ok(Some(record)) = network.get_local_record(&record_key).await { - result = Ok((our_address, Bytes::from(record.value))); - } - } - - QueryResponse::GetRegisterRecord(result) - } - Query::GetReplicatedRecord { requester, key } => { - debug!("Got GetReplicatedRecord from {requester:?} regarding {key:?}"); - + Query::GetReplicatedRecord { requester: _, key } => { let our_address = NetworkAddress::from_peer(network.peer_id()); let mut result = Err(ProtocolError::ReplicatedRecordNotFound { holder: Box::new(our_address.clone()), @@ -668,16 +677,9 @@ impl Node { key, nonce, difficulty, - } => { - debug!( - "Got GetChunkExistenceProof targeting chunk {key:?} with {difficulty} answers." - ); - - QueryResponse::GetChunkExistenceProof( - Self::respond_x_closest_record_proof(network, key, nonce, difficulty, true) - .await, - ) - } + } => QueryResponse::GetChunkExistenceProof( + Self::respond_x_closest_record_proof(network, key, nonce, difficulty, true).await, + ), Query::CheckNodeInProblem(target_address) => { debug!("Got CheckNodeInProblem for peer {target_address:?}"); @@ -752,12 +754,12 @@ impl Node { ) -> Vec<(NetworkAddress, Vec)> { match (num_of_peers, range) { (_, Some(value)) => { - let distance = U256::from_be_bytes(value); + let distance = U256::from_big_endian(&value); peer_addrs .iter() .filter_map(|(peer_id, multi_addrs)| { let addr = NetworkAddress::from_peer(*peer_id); - if convert_distance_to_u256(&target.distance(&addr)) <= distance { + if target.distance(&addr).0 <= distance { Some((addr, multi_addrs.clone())) } else { None @@ -811,7 +813,7 @@ impl Node { all_local_records .iter() .filter_map(|(addr, record_type)| { - if *record_type == RecordType::Chunk { + if *record_type == ValidationType::Chunk { Some(addr.clone()) } else { None @@ -837,12 +839,12 @@ impl Node { } } } - } - info!( - "Respond with {} answers to the StorageChallenge targeting {key:?} with {difficulty} difficulty, in {:?}", - results.len(), start.elapsed() - ); + info!( + "Respond with {} answers to the StorageChallenge targeting {key:?} with {difficulty} difficulty, in {:?}", + results.len(), start.elapsed() + ); + } results } @@ -876,7 +878,7 @@ impl Node { all_keys .iter() .filter_map(|(addr, record_type)| { - if RecordType::Chunk == *record_type { + if ValidationType::Chunk == *record_type { Some(addr.clone()) } else { None @@ -935,20 +937,26 @@ impl Node { }); } + let mut peer_scores = vec![]; while let Some(res) = tasks.join_next().await { match res { Ok((peer_id, score)) => { - if score < MIN_ACCEPTABLE_HEALTHY_SCORE { + let is_healthy = score > MIN_ACCEPTABLE_HEALTHY_SCORE; + if !is_healthy { info!("Peer {peer_id:?} failed storage challenge with low score {score}/{MIN_ACCEPTABLE_HEALTHY_SCORE}."); // TODO: shall the challenge failure immediately triggers the node to be removed? network.record_node_issues(peer_id, NodeIssue::FailedChunkProofCheck); } + peer_scores.push((peer_id, is_healthy)); } Err(e) => { info!("StorageChallenge task completed with error {e:?}"); } } } + if !peer_scores.is_empty() { + network.notify_peer_scores(peer_scores); + } info!( "Completed node StorageChallenge against neighbours in {:?}!", @@ -1158,12 +1166,12 @@ mod tests { ); // Range shall be preferred, i.e. the result peers shall all within the range - let distance = U256::from_be_bytes(range_value); + let distance = U256::from_big_endian(&range_value); let expected_result: Vec<(NetworkAddress, Vec)> = local_peers .into_iter() .filter_map(|(peer_id, multi_addrs)| { let addr = NetworkAddress::from_peer(peer_id); - if convert_distance_to_u256(&target.distance(&addr)) <= distance { + if target.distance(&addr).0 <= distance { Some((addr, multi_addrs.clone())) } else { None diff --git a/ant-node/src/put_validation.rs b/ant-node/src/put_validation.rs index 67a01b275b..f327045d94 100644 --- a/ant-node/src/put_validation.rs +++ b/ant-node/src/put_validation.rs @@ -10,17 +10,16 @@ use std::collections::BTreeSet; use crate::{node::Node, Error, Marker, Result}; use ant_evm::payment_vault::verify_data_payment; -use ant_evm::{AttoTokens, ProofOfPayment}; +use ant_evm::ProofOfPayment; use ant_networking::NetworkError; -use ant_protocol::storage::Transaction; +use ant_protocol::storage::GraphEntry; use ant_protocol::{ storage::{ - try_deserialize_record, try_serialize_record, Chunk, RecordHeader, RecordKind, RecordType, - Scratchpad, TransactionAddress, + try_deserialize_record, try_serialize_record, Chunk, DataTypes, GraphEntryAddress, Pointer, + PointerAddress, RecordHeader, RecordKind, Scratchpad, ValidationType, }, NetworkAddress, PrettyPrintRecordKey, }; -use ant_registers::SignedRegister; use libp2p::kad::{Record, RecordKey}; use xor_name::XorName; @@ -30,7 +29,7 @@ impl Node { let record_header = RecordHeader::from_record(&record)?; match record_header.kind { - RecordKind::ChunkWithPayment => { + RecordKind::DataWithPayment(DataTypes::Chunk) => { let record_key = record.key.clone(); let (payment, chunk) = try_deserialize_record::<(ProofOfPayment, Chunk)>(&record)?; let already_exists = self @@ -40,22 +39,30 @@ impl Node { // Validate the payment and that we received what we asked. // This stores any payments to disk let payment_res = self - .payment_for_us_exists_and_is_still_valid(&chunk.network_address(), payment) + .payment_for_us_exists_and_is_still_valid( + &chunk.network_address(), + DataTypes::Chunk, + payment.clone(), + ) .await; // Now that we've taken any money passed to us, regardless of the payment's validity, // if we already have the data we can return early if already_exists { - // if we're receiving this chunk PUT again, and we have been paid, - // we eagerly retry replicaiton as it seems like other nodes are having trouble - // did not manage to get this chunk as yet - self.replicate_valid_fresh_record(record_key, RecordType::Chunk); + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // self.replicate_valid_fresh_record( + // record_key, + // DataTypes::Chunk, + // ValidationType::Chunk, + // Some(payment), + // ); // Notify replication_fetcher to mark the attempt as completed. // Send the notification earlier to avoid it got skipped due to: // the record becomes stored during the fetch because of other interleaved process. self.network() - .notify_fetch_completed(record.key.clone(), RecordType::Chunk); + .notify_fetch_completed(record.key.clone(), ValidationType::Chunk); debug!( "Chunk with addr {:?} already exists: {already_exists}, payment extracted.", @@ -70,30 +77,37 @@ impl Node { // Writing chunk to disk takes time, hence try to execute it first. // So that when the replicate target asking for the copy, // the node can have a higher chance to respond. - let store_chunk_result = self.store_chunk(&chunk); + let store_chunk_result = self.store_chunk(&chunk, true); if store_chunk_result.is_ok() { Marker::ValidPaidChunkPutFromClient(&PrettyPrintRecordKey::from(&record.key)) .log(); - self.replicate_valid_fresh_record(record_key, RecordType::Chunk); + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // self.replicate_valid_fresh_record( + // record_key, + // DataTypes::Chunk, + // ValidationType::Chunk, + // Some(payment), + // ); // Notify replication_fetcher to mark the attempt as completed. // Send the notification earlier to avoid it got skipped due to: // the record becomes stored during the fetch because of other interleaved process. self.network() - .notify_fetch_completed(record.key.clone(), RecordType::Chunk); + .notify_fetch_completed(record.key.clone(), ValidationType::Chunk); } store_chunk_result } - RecordKind::Chunk => { + RecordKind::DataOnly(DataTypes::Chunk) => { error!("Chunk should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::ScratchpadWithPayment => { + RecordKind::DataWithPayment(DataTypes::Scratchpad) => { let record_key = record.key.clone(); let (payment, scratchpad) = try_deserialize_record::<(ProofOfPayment, Scratchpad)>(&record)?; @@ -106,7 +120,8 @@ impl Node { let payment_res = self .payment_for_us_exists_and_is_still_valid( &scratchpad.network_address(), - payment, + DataTypes::Scratchpad, + payment.clone(), ) .await; @@ -117,7 +132,12 @@ impl Node { // So that when the replicate target asking for the copy, // the node can have a higher chance to respond. let store_scratchpad_result = self - .validate_and_store_scratchpad_record(scratchpad, record_key.clone(), true) + .validate_and_store_scratchpad_record( + scratchpad, + record_key.clone(), + true, + Some(payment), + ) .await; match store_scratchpad_result { @@ -125,27 +145,26 @@ impl Node { // we eagerly retry replicaiton as it seems like other nodes are having trouble // did not manage to get this scratchpad as yet. Ok(_) | Err(Error::IgnoringOutdatedScratchpadPut) => { + let content_hash = XorName::from_content(&record.value); Marker::ValidScratchpadRecordPutFromClient(&PrettyPrintRecordKey::from( &record_key, )) .log(); - self.replicate_valid_fresh_record( - record_key.clone(), - RecordType::Scratchpad, - ); // Notify replication_fetcher to mark the attempt as completed. // Send the notification earlier to avoid it got skipped due to: // the record becomes stored during the fetch because of other interleaved process. - self.network() - .notify_fetch_completed(record_key, RecordType::Scratchpad); + self.network().notify_fetch_completed( + record_key, + ValidationType::NonChunk(content_hash), + ); } Err(_) => {} } store_scratchpad_result } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { // make sure we already have this scratchpad locally, else reject it as first time upload needs payment let key = record.key.clone(); let scratchpad = try_deserialize_record::(&record)?; @@ -160,153 +179,153 @@ impl Node { } // store the scratchpad - self.validate_and_store_scratchpad_record(scratchpad, key, false) + self.validate_and_store_scratchpad_record(scratchpad, key, true, None) .await } - RecordKind::Transaction => { + RecordKind::DataOnly(DataTypes::GraphEntry) => { // Transactions should always be paid for error!("Transaction should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::TransactionWithPayment => { - let (payment, transaction) = - try_deserialize_record::<(ProofOfPayment, Transaction)>(&record)?; + RecordKind::DataWithPayment(DataTypes::GraphEntry) => { + let (payment, graph_entry) = + try_deserialize_record::<(ProofOfPayment, GraphEntry)>(&record)?; - // check if the deserialized value's TransactionAddress matches the record's key - let net_addr = NetworkAddress::from_transaction_address(transaction.address()); + // check if the deserialized value's GraphEntryAddress matches the record's key + let net_addr = NetworkAddress::from_graph_entry_address(graph_entry.address()); let key = net_addr.to_record_key(); let pretty_key = PrettyPrintRecordKey::from(&key); if record.key != key { warn!( - "Record's key {pretty_key:?} does not match with the value's TransactionAddress, ignoring PUT." + "Record's key {pretty_key:?} does not match with the value's GraphEntryAddress, ignoring PUT." ); return Err(Error::RecordKeyMismatch); } let already_exists = self.validate_key_and_existence(&net_addr, &key).await?; - // The transaction may already exist during the replication. - // The payment shall get deposit to self even the transaction already presents. - // However, if the transaction is already present, the incoming one shall be + // The GraphEntry may already exist during the replication. + // The payment shall get deposit to self even the GraphEntry already presents. + // However, if the GraphEntry is already present, the incoming one shall be // appended with the existing one, if content is different. if let Err(err) = self - .payment_for_us_exists_and_is_still_valid(&net_addr, payment) + .payment_for_us_exists_and_is_still_valid( + &net_addr, + DataTypes::GraphEntry, + payment.clone(), + ) .await { if already_exists { - debug!("Payment of the incoming exists transaction {pretty_key:?} having error {err:?}"); + debug!("Payment of the incoming existing GraphEntry {pretty_key:?} having error {err:?}"); } else { - error!("Payment of the incoming non-exist transaction {pretty_key:?} having error {err:?}"); + error!("Payment of the incoming new GraphEntry {pretty_key:?} having error {err:?}"); return Err(err); } } let res = self - .validate_merge_and_store_transactions(vec![transaction], &key) + .validate_merge_and_store_graphentries(vec![graph_entry], &key, true) .await; if res.is_ok() { let content_hash = XorName::from_content(&record.value); - Marker::ValidTransactionPutFromClient(&PrettyPrintRecordKey::from(&record.key)) + Marker::ValidGraphEntryPutFromClient(&PrettyPrintRecordKey::from(&record.key)) .log(); - self.replicate_valid_fresh_record( - record.key.clone(), - RecordType::NonChunk(content_hash), - ); + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // self.replicate_valid_fresh_record( + // record.key.clone(), + // DataTypes::GraphEntry, + // ValidationType::NonChunk(content_hash), + // Some(payment), + // ); // Notify replication_fetcher to mark the attempt as completed. // Send the notification earlier to avoid it got skipped due to: // the record becomes stored during the fetch because of other interleaved process. self.network().notify_fetch_completed( record.key.clone(), - RecordType::NonChunk(content_hash), + ValidationType::NonChunk(content_hash), ); } res } - RecordKind::Register => { - let register = try_deserialize_record::(&record)?; + RecordKind::DataOnly(DataTypes::Pointer) => { + let pointer = try_deserialize_record::(&record)?; + let net_addr = NetworkAddress::from_pointer_address(pointer.address()); + let pretty_key = PrettyPrintRecordKey::from(&record.key); + let already_exists = self + .validate_key_and_existence(&net_addr, &record.key) + .await?; - // make sure we already have this register locally - let net_addr = NetworkAddress::from_register_address(*register.address()); - let key = net_addr.to_record_key(); - let pretty_key = PrettyPrintRecordKey::from(&key); - debug!("Got record to store without payment for register at {pretty_key:?}"); - if !self.validate_key_and_existence(&net_addr, &key).await? { - debug!("Ignore store without payment for register at {pretty_key:?}"); + if !already_exists { + warn!("Pointer at address: {:?}, key: {:?} does not exist locally, rejecting PUT without payment", pointer.address(), pretty_key); return Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )); } - // store the update - debug!("Store update without payment as we already had register at {pretty_key:?}"); - let result = self.validate_and_store_register(register, true).await; - - if result.is_ok() { - debug!("Successfully stored register update at {pretty_key:?}"); - Marker::ValidPaidRegisterPutFromClient(&pretty_key).log(); - // we dont try and force replicaiton here as there's state to be kept in sync - // which we leave up to the client to enforce - + let res = self + .validate_and_store_pointer_record(pointer, record.key.clone(), true, None) + .await; + if res.is_ok() { let content_hash = XorName::from_content(&record.value); + Marker::ValidPointerPutFromClient(&pretty_key).log(); // Notify replication_fetcher to mark the attempt as completed. - // Send the notification earlier to avoid it got skipped due to: - // the record becomes stored during the fetch because of other interleaved process. self.network().notify_fetch_completed( record.key.clone(), - RecordType::NonChunk(content_hash), + ValidationType::NonChunk(content_hash), ); - } else { - warn!("Failed to store register update at {pretty_key:?}"); } - result + res } - RecordKind::RegisterWithPayment => { - let (payment, register) = - try_deserialize_record::<(ProofOfPayment, SignedRegister)>(&record)?; - - // check if the deserialized value's RegisterAddress matches the record's key - let net_addr = NetworkAddress::from_register_address(*register.address()); - let key = net_addr.to_record_key(); - let pretty_key = PrettyPrintRecordKey::from(&key); - if record.key != key { - warn!( - "Record's key {pretty_key:?} does not match with the value's RegisterAddress, ignoring PUT." - ); - return Err(Error::RecordKeyMismatch); - } + RecordKind::DataWithPayment(DataTypes::Pointer) => { + let (payment, pointer) = + try_deserialize_record::<(ProofOfPayment, Pointer)>(&record)?; - let already_exists = self.validate_key_and_existence(&net_addr, &key).await?; + let net_addr = NetworkAddress::from_pointer_address(pointer.address()); + let pretty_key = PrettyPrintRecordKey::from(&record.key); + let already_exists = self + .validate_key_and_existence(&net_addr, &record.key) + .await?; - // The register may already exist during the replication. - // The payment shall get deposit to self even the register already presents. - // However, if the register already presents, the incoming one maybe for edit only. - // Hence the corresponding payment error shall not be thrown out. + // The pointer may already exist during the replication. + // The payment shall get deposit to self even if the pointer already exists. if let Err(err) = self - .payment_for_us_exists_and_is_still_valid(&net_addr, payment) + .payment_for_us_exists_and_is_still_valid( + &net_addr, + DataTypes::Pointer, + payment.clone(), + ) .await { if already_exists { - debug!("Payment of the incoming exists register {pretty_key:?} having error {err:?}"); + debug!("Payment of the incoming exists pointer {pretty_key:?} having error {err:?}"); } else { - error!("Payment of the incoming non-exist register {pretty_key:?} having error {err:?}"); + error!("Payment of the incoming non-exist pointer {pretty_key:?} having error {err:?}"); return Err(err); } } - let res = self.validate_and_store_register(register, true).await; + let res = self + .validate_and_store_pointer_record( + pointer, + record.key.clone(), + true, + Some(payment), + ) + .await; if res.is_ok() { let content_hash = XorName::from_content(&record.value); + Marker::ValidPointerPutFromClient(&pretty_key).log(); // Notify replication_fetcher to mark the attempt as completed. - // Send the notification earlier to avoid it got skipped due to: - // the record becomes stored during the fetch because of other interleaved process. self.network().notify_fetch_completed( record.key.clone(), - RecordType::NonChunk(content_hash), + ValidationType::NonChunk(content_hash), ); } res @@ -316,20 +335,20 @@ impl Node { /// Store a pre-validated, and already paid record to the RecordStore pub(crate) async fn store_replicated_in_record(&self, record: Record) -> Result<()> { - debug!("Storing record which was replicated to us {:?}", record.key); + debug!( + "Storing record which was replicated to us {:?}", + PrettyPrintRecordKey::from(&record.key) + ); let record_header = RecordHeader::from_record(&record)?; match record_header.kind { - // A separate flow handles payment for chunks and registers - RecordKind::ChunkWithPayment - | RecordKind::TransactionWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::ScratchpadWithPayment => { + // A separate flow handles record with payment + RecordKind::DataWithPayment(_) => { warn!("Prepaid record came with Payment, which should be handled in another flow"); Err(Error::UnexpectedRecordWithPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::Chunk => { + RecordKind::DataOnly(DataTypes::Chunk) => { let chunk = try_deserialize_record::(&record)?; let record_key = record.key.clone(); @@ -344,40 +363,32 @@ impl Node { return Ok(()); } - self.store_chunk(&chunk) + self.store_chunk(&chunk, false) } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { let key = record.key.clone(); let scratchpad = try_deserialize_record::(&record)?; - self.validate_and_store_scratchpad_record(scratchpad, key, false) + self.validate_and_store_scratchpad_record(scratchpad, key, false, None) .await } - RecordKind::Transaction => { + RecordKind::DataOnly(DataTypes::GraphEntry) => { let record_key = record.key.clone(); - let transactions = try_deserialize_record::>(&record)?; - self.validate_merge_and_store_transactions(transactions, &record_key) + let graph_entries = try_deserialize_record::>(&record)?; + self.validate_merge_and_store_graphentries(graph_entries, &record_key, false) .await } - RecordKind::Register => { - let register = try_deserialize_record::(&record)?; - - // check if the deserialized value's RegisterAddress matches the record's key - let key = - NetworkAddress::from_register_address(*register.address()).to_record_key(); - if record.key != key { - warn!( - "Record's key does not match with the value's RegisterAddress, ignoring PUT." - ); - return Err(Error::RecordKeyMismatch); - } - self.validate_and_store_register(register, false).await + RecordKind::DataOnly(DataTypes::Pointer) => { + let pointer = try_deserialize_record::(&record)?; + let key = record.key.clone(); + self.validate_and_store_pointer_record(pointer, key, false, None) + .await } } } /// Check key is valid compared to the network name, and if we already have this data or not. /// returns true if data already exists locally - async fn validate_key_and_existence( + pub(crate) async fn validate_key_and_existence( &self, address: &NetworkAddress, expected_record_key: &RecordKey, @@ -413,28 +424,25 @@ impl Node { } /// Store a `Chunk` to the RecordStore - pub(crate) fn store_chunk(&self, chunk: &Chunk) -> Result<()> { - let chunk_name = *chunk.name(); - let chunk_addr = *chunk.address(); - + pub(crate) fn store_chunk(&self, chunk: &Chunk, is_client_put: bool) -> Result<()> { let key = NetworkAddress::from_chunk_address(*chunk.address()).to_record_key(); let pretty_key = PrettyPrintRecordKey::from(&key).into_owned(); let record = Record { key, - value: try_serialize_record(&chunk, RecordKind::Chunk)?.to_vec(), + value: try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk))?.to_vec(), publisher: None, expires: None, }; // finally store the Record directly into the local storage - debug!("Storing chunk {chunk_name:?} as Record locally"); - self.network().put_local_record(record); + self.network().put_local_record(record, is_client_put); self.record_metrics(Marker::ValidChunkRecordPutFromNetwork(&pretty_key)); - self.events_channel() - .broadcast(crate::NodeEvent::ChunkStored(chunk_addr)); + // TODO: currently ignored, re-enable once start to handle + // self.events_channel() + // .broadcast(crate::NodeEvent::ChunkStored(chunk_addr)); Ok(()) } @@ -451,13 +459,14 @@ impl Node { scratchpad: Scratchpad, record_key: RecordKey, is_client_put: bool, + _payment: Option, ) -> Result<()> { // owner PK is defined herein, so as long as record key and this match, we're good let addr = scratchpad.address(); - let count = scratchpad.count(); + let count = scratchpad.counter(); debug!("Validating and storing scratchpad {addr:?} with count {count}"); - // check if the deserialized value's RegisterAddress matches the record's key + // check if the deserialized value's ScratchpadAddress matches the record's key let scratchpad_key = NetworkAddress::ScratchpadAddress(*addr).to_record_key(); if scratchpad_key != record_key { warn!("Record's key does not match with the value's ScratchpadAddress, ignoring PUT."); @@ -467,18 +476,24 @@ impl Node { // check if the Scratchpad is present locally that we don't have a newer version if let Some(local_pad) = self.network().get_local_record(&scratchpad_key).await? { let local_pad = try_deserialize_record::(&local_pad)?; - if local_pad.count() >= scratchpad.count() { + if local_pad.counter() >= scratchpad.counter() { warn!("Rejecting Scratchpad PUT with counter less than or equal to the current counter"); return Err(Error::IgnoringOutdatedScratchpadPut); } } // ensure data integrity - if !scratchpad.is_valid() { + if !scratchpad.verify_signature() { warn!("Rejecting Scratchpad PUT with invalid signature"); return Err(Error::InvalidScratchpadSignature); } + // ensure the scratchpad is not too big + if scratchpad.is_too_big() { + warn!("Rejecting Scratchpad PUT with too big size"); + return Err(Error::ScratchpadTooBig(scratchpad.size())); + } + info!( "Storing sratchpad {addr:?} with content of {:?} as Record locally", scratchpad.encrypted_data_hash() @@ -486,161 +501,133 @@ impl Node { let record = Record { key: scratchpad_key.clone(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad)?.to_vec(), + value: try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad))? + .to_vec(), publisher: None, expires: None, }; - self.network().put_local_record(record); + self.network() + .put_local_record(record.clone(), is_client_put); let pretty_key = PrettyPrintRecordKey::from(&scratchpad_key); self.record_metrics(Marker::ValidScratchpadRecordPutFromNetwork(&pretty_key)); - if is_client_put { - self.replicate_valid_fresh_record(scratchpad_key, RecordType::Scratchpad); - } + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // if is_client_put { + // let content_hash = XorName::from_content(&record.value); + // // ScratchPad update is a special upload that without payment, + // // but must have an existing copy to update. + // self.replicate_valid_fresh_record( + // scratchpad_key, + // DataTypes::Scratchpad, + // ValidationType::NonChunk(content_hash), + // payment, + // ); + // } Ok(()) } - /// Validate and store a `Register` to the RecordStore - pub(crate) async fn validate_and_store_register( - &self, - register: SignedRegister, - is_client_put: bool, - ) -> Result<()> { - let reg_addr = register.address(); - debug!("Validating and storing register {reg_addr:?}"); - - // check if the Register is present locally - let key = NetworkAddress::from_register_address(*reg_addr).to_record_key(); - let present_locally = self.network().is_record_key_present_locally(&key).await?; - let pretty_key = PrettyPrintRecordKey::from(&key); - - // check register and merge if needed - let updated_register = match self.register_validation(®ister, present_locally).await? { - Some(reg) => { - debug!("Register {pretty_key:?} needed to be updated"); - reg - } - None => { - debug!("No update needed for register"); - return Ok(()); - } - }; - // store in kad - let record = Record { - key: key.clone(), - value: try_serialize_record(&updated_register, RecordKind::Register)?.to_vec(), - publisher: None, - expires: None, - }; - let content_hash = XorName::from_content(&record.value); - - info!("Storing register {reg_addr:?} with content of {content_hash:?} as Record locally"); - self.network().put_local_record(record); - - self.record_metrics(Marker::ValidRegisterRecordPutFromNetwork(&pretty_key)); - - // Updated register needs to be replicated out as well, - // to avoid `leaking` of old version due to the mismatch of - // `close_range` and `replication_range`, combined with nodes churning - // - // However, to avoid `looping of replication`, a `replicated in` register - // shall not trigger any further replication out. - if is_client_put { - self.replicate_valid_fresh_record(key, RecordType::NonChunk(content_hash)); - } - - Ok(()) - } - - /// Validate and store `Vec` to the RecordStore - /// If we already have a transaction at this address, the Vec is extended and stored. - pub(crate) async fn validate_merge_and_store_transactions( + /// Validate and store `Vec` to the RecordStore + /// If we already have a GraphEntry at this address, the Vec is extended and stored. + pub(crate) async fn validate_merge_and_store_graphentries( &self, - transactions: Vec, + entries: Vec, record_key: &RecordKey, + is_client_put: bool, ) -> Result<()> { let pretty_key = PrettyPrintRecordKey::from(record_key); - debug!("Validating transactions before storage at {pretty_key:?}"); + debug!("Validating GraphEntries before storage at {pretty_key:?}"); - // only keep transactions that match the record key - let transactions_for_key: Vec = transactions + // only keep GraphEntries that match the record key + let entries_for_key: Vec = entries .into_iter() .filter(|s| { - // get the record key for the transaction - let transaction_address = s.address(); - let network_address = NetworkAddress::from_transaction_address(transaction_address); - let transaction_record_key = network_address.to_record_key(); - let transaction_pretty = PrettyPrintRecordKey::from(&transaction_record_key); - if &transaction_record_key != record_key { - warn!("Ignoring transaction for another record key {transaction_pretty:?} when verifying: {pretty_key:?}"); + // get the record key for the GraphEntry + let graph_entry_address = s.address(); + let network_address = NetworkAddress::from_graph_entry_address(graph_entry_address); + let graph_entry_record_key = network_address.to_record_key(); + let graph_entry_pretty = PrettyPrintRecordKey::from(&graph_entry_record_key); + if &graph_entry_record_key != record_key { + warn!("Ignoring GraphEntry for another record key {graph_entry_pretty:?} when verifying: {pretty_key:?}"); return false; } true }) .collect(); - // if we have no transactions to verify, return early - if transactions_for_key.is_empty() { - warn!("Found no valid transactions to verify upon validation for {pretty_key:?}"); + // if we have no GraphEntries to verify, return early + if entries_for_key.is_empty() { + warn!("Found no valid GraphEntries to verify upon validation for {pretty_key:?}"); return Err(Error::InvalidRequest(format!( - "No transactions to verify when validating {pretty_key:?}" + "No GraphEntries to verify when validating {pretty_key:?}" ))); } - // verify the transactions - let mut validated_transactions: BTreeSet = transactions_for_key + // verify the GraphEntries + let mut validated_entries: BTreeSet = entries_for_key .into_iter() - .filter(|t| t.verify()) + .filter(|t| t.verify_signature()) .collect(); // skip if none are valid - let addr = match validated_transactions.first() { + let addr = match validated_entries.first() { None => { - warn!("Found no validated transactions to store at {pretty_key:?}"); + warn!("Found no validated GraphEntries to store at {pretty_key:?}"); return Ok(()); } Some(t) => t.address(), }; - // add local transactions to the validated transactions, turn to Vec - let local_txs = self.get_local_transactions(addr).await?; - validated_transactions.extend(local_txs.into_iter()); - let validated_transactions: Vec = validated_transactions.into_iter().collect(); + // add local GraphEntries to the validated GraphEntries, turn to Vec + let local_entries = self.get_local_graphentries(addr).await?; + let existing_entry = local_entries.len(); + validated_entries.extend(local_entries.into_iter()); + let validated_entries: Vec = validated_entries.into_iter().collect(); + + // No need to write to disk if nothing new. + if existing_entry == validated_entries.len() { + debug!("No new entry of the GraphEntry {pretty_key:?}"); + return Ok(()); + } // store the record into the local storage let record = Record { key: record_key.clone(), - value: try_serialize_record(&validated_transactions, RecordKind::Transaction)?.to_vec(), + value: try_serialize_record( + &validated_entries, + RecordKind::DataOnly(DataTypes::GraphEntry), + )? + .to_vec(), publisher: None, expires: None, }; - self.network().put_local_record(record); - debug!("Successfully stored validated transactions at {pretty_key:?}"); + self.network().put_local_record(record, is_client_put); + debug!("Successfully stored validated GraphEntries at {pretty_key:?}"); - // Just log the multiple transactions - if validated_transactions.len() > 1 { + // Just log the multiple GraphEntries + if validated_entries.len() > 1 { debug!( - "Got multiple transaction(s) of len {} at {pretty_key:?}", - validated_transactions.len() + "Got multiple GraphEntry(s) of len {} at {pretty_key:?}", + validated_entries.len() ); } - self.record_metrics(Marker::ValidTransactionRecordPutFromNetwork(&pretty_key)); + self.record_metrics(Marker::ValidGraphEntryRecordPutFromNetwork(&pretty_key)); Ok(()) } /// Perform validations on the provided `Record`. - async fn payment_for_us_exists_and_is_still_valid( + pub(crate) async fn payment_for_us_exists_and_is_still_valid( &self, address: &NetworkAddress, + data_type: DataTypes, payment: ProofOfPayment, ) -> Result<()> { let key = address.to_record_key(); let pretty_key = PrettyPrintRecordKey::from(&key).into_owned(); - debug!("Validating record payment for {pretty_key}"); // check if the quote is valid let self_peer_id = self.network().peer_id(); @@ -650,7 +637,6 @@ impl Node { "Payment is not valid for record {pretty_key}" ))); } - debug!("Payment is valid for record {pretty_key}"); // verify quote expiration if payment.has_expired() { @@ -660,11 +646,20 @@ impl Node { ))); } + // verify data type matches + if !payment.verify_data_type(data_type.get_index()) { + warn!("Payment quote has wrong data type for record {pretty_key}"); + return Err(Error::InvalidRequest(format!( + "Payment quote has wrong data type for record {pretty_key}" + ))); + } + // verify the claimed payees are all known to us within the certain range. let closest_k_peers = self.network().get_closest_k_value_local_peers().await?; let mut payees = payment.payees(); payees.retain(|peer_id| !closest_k_peers.contains(peer_id)); if !payees.is_empty() { + warn!("Payment quote has out-of-range payees for record {pretty_key}"); return Err(Error::InvalidRequest(format!( "Payment quote has out-of-range payees {payees:?}" ))); @@ -677,113 +672,184 @@ impl Node { .collect(); // check if payment is valid on chain let payments_to_verify = payment.digest(); - debug!("Verifying payment for record {pretty_key}"); let reward_amount = verify_data_payment(self.evm_network(), owned_payment_quotes, payments_to_verify) .await - .map_err(|e| Error::EvmNetwork(format!("Failed to verify chunk payment: {e}")))?; + .inspect_err(|e| { + warn!("Failed to verify record payment: {e}"); + }) + .map_err(|e| Error::EvmNetwork(format!("Failed to verify record payment: {e}")))?; + debug!("Payment of {reward_amount:?} is valid for record {pretty_key}"); - // Notify `record_store` that the node received a payment. - self.network().notify_payment_received(); - - #[cfg(feature = "open-metrics")] - if let Some(metrics_recorder) = self.metrics_recorder() { - // FIXME: We would reach the MAX if the storecost is scaled up. - let current_value = metrics_recorder.current_reward_wallet_balance.get(); - let new_value = - current_value.saturating_add(reward_amount.try_into().unwrap_or(i64::MAX)); - let _ = metrics_recorder - .current_reward_wallet_balance - .set(new_value); - } - self.events_channel() - .broadcast(crate::NodeEvent::RewardReceived( - AttoTokens::from(reward_amount), - address.clone(), - )); - - // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): - info!("Total payment of {reward_amount:?} atto tokens accepted for record {pretty_key}"); - - // loud mode: print a celebratory message to console - #[cfg(feature = "loud")] - { - println!("🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟 RECEIVED REWARD 🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟"); - println!( + if !reward_amount.is_zero() { + // Notify `record_store` that the node received a payment. + self.network().notify_payment_received(); + + #[cfg(feature = "open-metrics")] + if let Some(metrics_recorder) = self.metrics_recorder() { + // FIXME: We would reach the MAX if the storecost is scaled up. + let current_value = metrics_recorder.current_reward_wallet_balance.get(); + let new_value = + current_value.saturating_add(reward_amount.try_into().unwrap_or(i64::MAX)); + let _ = metrics_recorder + .current_reward_wallet_balance + .set(new_value); + } + + // TODO: currently ignored, re-enable once going to handle this. + // self.events_channel() + // .broadcast(crate::NodeEvent::RewardReceived( + // AttoTokens::from(reward_amount), + // address.clone(), + // )); + + // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): + info!( "Total payment of {reward_amount:?} atto tokens accepted for record {pretty_key}" ); - println!("🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟"); + + // loud mode: print a celebratory message to console + #[cfg(feature = "loud")] + { + println!("🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟 RECEIVED REWARD 🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟"); + println!( + "Total payment of {reward_amount:?} atto tokens accepted for record {pretty_key}" + ); + println!("🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟🌟"); + } } Ok(()) } - async fn register_validation( - &self, - register: &SignedRegister, - present_locally: bool, - ) -> Result> { - // check if register is valid - let reg_addr = register.address(); - register.verify()?; - - // if we don't have it locally return it - if !present_locally { - debug!("Register with addr {reg_addr:?} is valid and doesn't exist locally"); - return Ok(Some(register.to_owned())); - } - debug!("Register with addr {reg_addr:?} exists locally, comparing with local version"); - - let key = NetworkAddress::from_register_address(*reg_addr).to_record_key(); - - // get local register - let maybe_record = self.network().get_local_record(&key).await?; - let record = match maybe_record { + /// Get the local GraphEntries for the provided `GraphEntryAddress` + /// This only fetches the GraphEntries from the local store and does not perform any network operations. + async fn get_local_graphentries(&self, addr: GraphEntryAddress) -> Result> { + // get the local GraphEntries + let record_key = NetworkAddress::from_graph_entry_address(addr).to_record_key(); + debug!("Checking for local GraphEntries with key: {record_key:?}"); + let local_record = match self.network().get_local_record(&record_key).await? { Some(r) => r, None => { - error!("Register with addr {reg_addr:?} already exists locally, but not found in local storage"); - return Err(Error::InvalidRequest(format!( - "Register with addr {reg_addr:?} claimed to be existing locally was not found" - ))); + debug!("GraphEntry is not present locally: {record_key:?}"); + return Ok(vec![]); } }; - let local_register: SignedRegister = try_deserialize_record(&record)?; - - // merge the two registers - let mut merged_register = local_register.clone(); - merged_register.verified_merge(register)?; - if merged_register == local_register { - debug!("Register with addr {reg_addr:?} is the same as the local version"); - Ok(None) - } else { - debug!("Register with addr {reg_addr:?} is different from the local version"); - Ok(Some(merged_register)) + + // deserialize the record and get the GraphEntries + let local_header = RecordHeader::from_record(&local_record)?; + let record_kind = local_header.kind; + if !matches!(record_kind, RecordKind::DataOnly(DataTypes::GraphEntry)) { + error!("Found a {record_kind} when expecting to find GraphEntry at {addr:?}"); + return Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly( + DataTypes::GraphEntry, + )) + .into()); } + let local_entries: Vec = try_deserialize_record(&local_record)?; + Ok(local_entries) } - /// Get the local transactions for the provided `TransactionAddress` - /// This only fetches the transactions from the local store and does not perform any network operations. - async fn get_local_transactions(&self, addr: TransactionAddress) -> Result> { - // get the local transactions - let record_key = NetworkAddress::from_transaction_address(addr).to_record_key(); - debug!("Checking for local transactions with key: {record_key:?}"); - let local_record = match self.network().get_local_record(&record_key).await? { - Some(r) => r, - None => { - debug!("Transaction is not present locally: {record_key:?}"); - return Ok(vec![]); + /// Get the local Pointer for the provided `PointerAddress` + /// This only fetches the Pointer from the local store and does not perform any network operations. + /// If the local Pointer is not present or corrupted, returns `None`. + async fn get_local_pointer(&self, addr: PointerAddress) -> Option { + // get the local Pointer + let record_key = NetworkAddress::from_pointer_address(addr).to_record_key(); + debug!("Checking for local Pointer with key: {record_key:?}"); + let local_record = match self.network().get_local_record(&record_key).await { + Ok(Some(r)) => r, + Ok(None) => { + debug!("Pointer is not present locally: {record_key:?}"); + return None; + } + Err(e) => { + error!("Failed to get Pointer record at {addr:?}: {e}"); + return None; } }; - // deserialize the record and get the transactions - let local_header = RecordHeader::from_record(&local_record)?; + // deserialize the record and get the Pointer + let local_header = match RecordHeader::from_record(&local_record) { + Ok(h) => h, + Err(_) => { + error!("Failed to deserialize Pointer record at {addr:?}"); + return None; + } + }; let record_kind = local_header.kind; - if !matches!(record_kind, RecordKind::Transaction) { - error!("Found a {record_kind} when expecting to find Spend at {addr:?}"); - return Err(NetworkError::RecordKindMismatch(RecordKind::Transaction).into()); + if !matches!(record_kind, RecordKind::DataOnly(DataTypes::Pointer)) { + error!("Found a {record_kind} when expecting to find Pointer at {addr:?}"); + return None; + } + let local_pointer: Pointer = match try_deserialize_record(&local_record) { + Ok(p) => p, + Err(_) => { + error!("Failed to deserialize Pointer record at {addr:?}"); + return None; + } + }; + Some(local_pointer) + } + + /// Validate and store a pointer record + pub(crate) async fn validate_and_store_pointer_record( + &self, + pointer: Pointer, + key: RecordKey, + is_client_put: bool, + _payment: Option, + ) -> Result<()> { + // Verify the pointer's signature + if !pointer.verify_signature() { + warn!("Pointer signature verification failed"); + return Err(Error::InvalidSignature); + } + + // Check if the pointer's address matches the record key + let net_addr = NetworkAddress::from_pointer_address(pointer.address()); + if key != net_addr.to_record_key() { + warn!("Pointer address does not match record key"); + return Err(Error::RecordKeyMismatch); + } + + // Keep the pointer with the highest counter + if let Some(local_pointer) = self.get_local_pointer(pointer.address()).await { + if pointer.counter() <= local_pointer.counter() { + info!( + "Ignoring Pointer PUT at {key:?} with counter less than or equal to the current counter ({} <= {})", + pointer.counter(), + local_pointer.counter() + ); + return Ok(()); + } } - let local_transactions: Vec = try_deserialize_record(&local_record)?; - Ok(local_transactions) + + // Store the pointer + let record = Record { + key: key.clone(), + value: try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer))? + .to_vec(), + publisher: None, + expires: None, + }; + self.network() + .put_local_record(record.clone(), is_client_put); + + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // if is_client_put { + // let content_hash = XorName::from_content(&record.value); + // self.replicate_valid_fresh_record( + // key.clone(), + // DataTypes::Pointer, + // ValidationType::NonChunk(content_hash), + // payment, + // ); + // } + + info!("Successfully stored Pointer record at {key:?}"); + Ok(()) } } diff --git a/ant-node/src/python.rs b/ant-node/src/python.rs index 3d50520940..d8f3dd9479 100644 --- a/ant-node/src/python.rs +++ b/ant-node/src/python.rs @@ -3,16 +3,12 @@ use crate::{NodeBuilder, RunningNode}; use ant_evm::{EvmNetwork, RewardsAddress}; -use ant_networking::PutRecordCfg; -use ant_protocol::{ - node::get_antnode_root_dir, - storage::{ChunkAddress, RecordType}, - NetworkAddress, -}; +use ant_networking::{PutRecordCfg, ResponseQuorum}; +use ant_protocol::{node::get_antnode_root_dir, storage::ChunkAddress, NetworkAddress}; use const_hex::FromHex; use libp2p::{ identity::{Keypair, PeerId}, - kad::{Quorum, Record as KadRecord}, + kad::Record as KadRecord, Multiaddr, }; use pyo3::{exceptions::PyRuntimeError, exceptions::PyValueError, prelude::*, types::PyModule}; @@ -104,7 +100,6 @@ impl AntNode { node_socket_addr, local, root_dir.unwrap_or_else(|| PathBuf::from(".")), - #[cfg(feature = "upnp")] false, ); node_builder.initial_peers(initial_peers); @@ -239,7 +234,7 @@ impl AntNode { self_: PyRef, key: String, value: Vec, - record_type: String, + _data_type: String, ) -> PyResult<()> { let node_guard = self_ .node @@ -250,12 +245,6 @@ impl AntNode { .try_lock() .map_err(|_| PyRuntimeError::new_err("Failed to acquire runtime lock"))?; - let _record_type = match record_type.to_lowercase().as_str() { - "chunk" => RecordType::Chunk, - "scratchpad" => RecordType::Scratchpad, - _ => return Err(PyValueError::new_err("Invalid record type. Must be one of: 'chunk', 'register', 'scratchpad', 'transaction'")), - }; - match (&*node_guard, &*rt_guard) { (Some(node), Some(rt)) => { let xorname = XorName::from_content( @@ -274,8 +263,8 @@ impl AntNode { expires: None, }; let cfg = PutRecordCfg { - put_quorum: Quorum::One, - retry_strategy: None, + put_quorum: ResponseQuorum::One, + retry_strategy: Default::default(), use_put_record_to: None, verification: None, }; @@ -417,6 +406,7 @@ impl AntNode { /// - Windows: C:\Users\\AppData\Roaming\autonomi\node\ #[allow(clippy::redundant_closure)] #[staticmethod] + #[pyo3(signature = (peer_id=None))] fn get_default_root_dir(peer_id: Option) -> PyResult { let peer_id = if let Some(id_str) = peer_id { let id = id_str @@ -478,7 +468,7 @@ impl AntNode { /// Python module initialization #[pymodule] #[pyo3(name = "_antnode")] -fn init_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/ant-node/src/quote.rs b/ant-node/src/quote.rs index f7c61b2af8..763016020c 100644 --- a/ant-node/src/quote.rs +++ b/ant-node/src/quote.rs @@ -12,6 +12,7 @@ use ant_networking::Network; use ant_protocol::{error::Error as ProtocolError, storage::ChunkAddress, NetworkAddress}; use libp2p::PeerId; use std::time::Duration; +use xor_name::XorName; impl Node { pub(crate) fn create_quote_for_storecost( @@ -20,7 +21,13 @@ impl Node { quoting_metrics: &QuotingMetrics, payment_address: &RewardsAddress, ) -> Result { - let content = address.as_xorname().unwrap_or_default(); + let content = match address { + NetworkAddress::ChunkAddress(addr) => *addr.xorname(), + NetworkAddress::GraphEntryAddress(addr) => *addr.xorname(), + NetworkAddress::ScratchpadAddress(addr) => addr.xorname(), + NetworkAddress::PointerAddress(addr) => *addr.xorname(), + NetworkAddress::PeerId(_) | NetworkAddress::RecordKey(_) => XorName::default(), + }; let timestamp = std::time::SystemTime::now(); let bytes = PaymentQuote::bytes_for_signing(content, timestamp, quoting_metrics, payment_address); @@ -38,7 +45,6 @@ impl Node { signature, }; - debug!("Created payment quote for {address:?}: {quote:?}"); Ok(quote) } } @@ -51,7 +57,14 @@ pub(crate) fn verify_quote_for_storecost( debug!("Verifying payment quote for {address:?}: {quote:?}"); // check address - if address.as_xorname().unwrap_or_default() != quote.content { + let content = match address { + NetworkAddress::ChunkAddress(addr) => *addr.xorname(), + NetworkAddress::GraphEntryAddress(addr) => *addr.xorname(), + NetworkAddress::ScratchpadAddress(addr) => addr.xorname(), + NetworkAddress::PointerAddress(addr) => *addr.xorname(), + NetworkAddress::PeerId(_) | NetworkAddress::RecordKey(_) => XorName::default(), + }; + if content != quote.content { return Err(Error::InvalidQuoteContent); } diff --git a/ant-node/src/replication.rs b/ant-node/src/replication.rs index 130b23e1f0..3d64c894cf 100644 --- a/ant-node/src/replication.rs +++ b/ant-node/src/replication.rs @@ -7,14 +7,16 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{error::Result, node::Node}; -use ant_networking::{GetRecordCfg, Network}; +use ant_evm::ProofOfPayment; +use ant_networking::{GetRecordCfg, Network, ResponseQuorum}; +use ant_protocol::storage::DataTypes; use ant_protocol::{ - messages::{Cmd, Query, QueryResponse, Request, Response}, - storage::RecordType, + messages::{Query, QueryResponse, Request, Response}, + storage::ValidationType, NetworkAddress, PrettyPrintRecordKey, }; use libp2p::{ - kad::{Quorum, Record, RecordKey}, + kad::{Record, RecordKey}, PeerId, }; use tokio::task::spawn; @@ -71,13 +73,10 @@ impl Node { "Can not fetch record {pretty_key:?} from node {holder:?}, fetching from the network" ); let get_cfg = GetRecordCfg { - get_quorum: Quorum::One, - retry_strategy: None, + get_quorum: ResponseQuorum::One, + retry_strategy: Default::default(), target_record: None, expected_holders: Default::default(), - // This is for replication, which doesn't have target_recrod to verify with. - // Hence value of the flag actually doesn't matter. - is_register: false, }; match node.network().get_record_from_network(key, &get_cfg).await { Ok(record) => record, @@ -101,80 +100,147 @@ impl Node { Ok(()) } - /// Replicate a fresh record to its close group peers. - /// This should not be triggered by a record we receive via replicaiton fetch - pub(crate) fn replicate_valid_fresh_record( + // Client changed to upload to ALL payees, hence no longer need this. + // May need again once client change back to upload to just one to save traffic. + // + // Replicate a fresh record to its close group peers. + // This should not be triggered by a record we receive via replicaiton fetch + // pub(crate) fn replicate_valid_fresh_record( + // &self, + // paid_key: RecordKey, + // data_type: DataTypes, + // validation_type: ValidationType, + // payment: Option, + // ) { + // let network = self.network().clone(); + + // let _handle = spawn(async move { + // let start = std::time::Instant::now(); + // let pretty_key = PrettyPrintRecordKey::from(&paid_key); + + // // first we wait until our own network store can return the record + // // otherwise it may not be fully written yet + // let mut retry_count = 0; + // debug!("Checking we have successfully stored the fresh record {pretty_key:?} in the store before replicating"); + // loop { + // let record = network.get_local_record(&paid_key).await.unwrap_or_else(|err| { + // error!( + // "Replicating fresh record {pretty_key:?} get_record_from_store errored: {err:?}" + // ); + // None + // }); + + // if record.is_some() { + // break; + // } + + // if retry_count > 10 { + // error!( + // "Could not get record from store for replication: {pretty_key:?} after 10 retries" + // ); + // return; + // } + + // retry_count += 1; + // tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // } + + // debug!("Start replication of fresh record {pretty_key:?} from store"); + + // let data_addr = NetworkAddress::from_record_key(&paid_key); + + // // If payment exists, only candidates are the payees. + // // Else get candidates from network. + // let replicate_candidates = match payment.as_ref() { + // Some(payment) => payment + // .payees() + // .into_iter() + // .filter(|peer_id| peer_id != &network.peer_id()) + // .collect(), + // None => match network.get_replicate_candidates(data_addr.clone()).await { + // Ok(peers) => peers, + // Err(err) => { + // error!("Replicating fresh record {pretty_key:?} get_replicate_candidates errored: {err:?}"); + // return; + // } + // }, + // }; + + // let our_peer_id = network.peer_id(); + // let our_address = NetworkAddress::from_peer(our_peer_id); + // let keys = vec![(data_addr, data_type, validation_type.clone(), payment)]; + + // for peer_id in replicate_candidates { + // debug!("Replicating fresh record {pretty_key:?} to {peer_id:?}"); + // let request = Request::Cmd(Cmd::FreshReplicate { + // holder: our_address.clone(), + // keys: keys.clone(), + // }); + + // network.send_req_ignore_reply(request, peer_id); + // } + // debug!( + // "Completed replicate fresh record {pretty_key:?} on store, in {:?}", + // start.elapsed() + // ); + // }); + // } + + // To fetch a received fresh record replication + pub(crate) fn fresh_replicate_to_fetch( &self, - paid_key: RecordKey, - record_type: RecordType, + holder: NetworkAddress, + keys: Vec<( + NetworkAddress, + DataTypes, + ValidationType, + Option, + )>, ) { - let network = self.network().clone(); - + let node = self.clone(); let _handle = spawn(async move { - let start = std::time::Instant::now(); - let pretty_key = PrettyPrintRecordKey::from(&paid_key); - - // first we wait until our own network store can return the record - // otherwise it may not be fully written yet - let mut retry_count = 0; - debug!("Checking we have successfully stored the fresh record {pretty_key:?} in the store before replicating"); - loop { - let record = match network.get_local_record(&paid_key).await { - Ok(record) => record, - Err(err) => { - error!( - "Replicating fresh record {pretty_key:?} get_record_from_store errored: {err:?}" - ); - None + let mut new_keys = vec![]; + for (addr, data_type, val_type, payment) in keys { + if let Some(payment) = payment { + // Payment must be valid + match node + .payment_for_us_exists_and_is_still_valid(&addr, data_type, payment) + .await + { + Ok(_) => {} + Err(err) => { + info!("ProofOfPayment of {addr:?} is invalid with error {err:?}"); + continue; + } + } + } else { + // Must have existing copy + match node + .validate_key_and_existence(&addr, &addr.to_record_key()) + .await + { + Ok(true) => {} + Ok(false) => { + info!( + "Received a fresh update against a non-existing record of {addr:?}" + ); + continue; + } + Err(err) => { + info!("Failed to verify the local existence of {addr:?} with error {err:?}"); + continue; + } } - }; - - if record.is_some() { - break; - } - - if retry_count > 10 { - error!( - "Could not get record from store for replication: {pretty_key:?} after 10 retries" - ); - return; } - - retry_count += 1; - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + new_keys.push((addr, val_type)); } - debug!("Start replication of fresh record {pretty_key:?} from store"); - - let data_addr = NetworkAddress::from_record_key(&paid_key); - let replicate_candidates = match network - .get_replicate_candidates(data_addr.clone()) - .await - { - Ok(peers) => peers, - Err(err) => { - error!("Replicating fresh record {pretty_key:?} get_replicate_candidates errored: {err:?}"); - return; - } - }; - - let our_peer_id = network.peer_id(); - let our_address = NetworkAddress::from_peer(our_peer_id); - let keys = vec![(data_addr, record_type.clone())]; - - for peer_id in replicate_candidates { - debug!("Replicating fresh record {pretty_key:?} to {peer_id:?}"); - let request = Request::Cmd(Cmd::Replicate { - holder: our_address.clone(), - keys: keys.clone(), - }); - - network.send_req_ignore_reply(request, peer_id); + if !new_keys.is_empty() { + // Adding to the replication_fetcher for the rate_limit purpose, + // instead of fetching directly. To reduce potential choking risk. + node.network() + .add_fresh_records_to_the_replication_fetcher(holder, new_keys); } - debug!( - "Completed replicate fresh record {pretty_key:?} on store, in {:?}", - start.elapsed() - ); }); } } diff --git a/ant-protocol/src/storage/address.rs b/ant-node/src/spawn/mod.rs similarity index 65% rename from ant-protocol/src/storage/address.rs rename to ant-node/src/spawn/mod.rs index 57c7a18aeb..88613bda2a 100644 --- a/ant-protocol/src/storage/address.rs +++ b/ant-node/src/spawn/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2024 MaidSafe.net limited. +// Copyright 2025 MaidSafe.net limited. // // This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. // Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed @@ -6,11 +6,5 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -mod chunk; -mod scratchpad; -mod transaction; - -pub use self::chunk::ChunkAddress; -pub use self::scratchpad::ScratchpadAddress; -pub use self::transaction::TransactionAddress; -pub use ant_registers::RegisterAddress; +pub mod network_spawner; +pub mod node_spawner; diff --git a/ant-node/src/spawn/network_spawner.rs b/ant-node/src/spawn/network_spawner.rs new file mode 100644 index 0000000000..78e36f8127 --- /dev/null +++ b/ant-node/src/spawn/network_spawner.rs @@ -0,0 +1,255 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::spawn::node_spawner::NodeSpawner; +use crate::RunningNode; +use ant_evm::{EvmNetwork, RewardsAddress}; +use libp2p::Multiaddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::PathBuf; + +pub struct NetworkSpawner { + /// The EVM network to which the spawned nodes will connect. + evm_network: EvmNetwork, + /// The address that will receive rewards from the spawned nodes. + rewards_address: RewardsAddress, + /// Specifies whether the network will operate in local mode and sets the listen address. + /// - `true`: Nodes listen on the local loopback address (`127.0.0.1`). + /// - `false`: Nodes listen on all available interfaces (`0.0.0.0`). + local: bool, + /// Enables or disables UPnP (automatic port forwarding). + upnp: bool, + /// Optional root directory to store node data and configurations. + root_dir: Option, + /// Number of nodes to spawn in the network. + size: usize, +} + +impl NetworkSpawner { + /// Creates a new `NetworkSpawner` with default configurations. + /// + /// Default values: + /// - `evm_network`: `EvmNetwork::default()` + /// - `rewards_address`: `RewardsAddress::default()` + /// - `local`: `false` + /// - `upnp`: `false` + /// - `root_dir`: `None` + /// - `size`: `5` + pub fn new() -> Self { + Self { + evm_network: Default::default(), + rewards_address: Default::default(), + local: false, + upnp: false, + root_dir: None, + size: 5, + } + } + + /// Sets the EVM network to be used by the nodes. + /// + /// # Arguments + /// + /// * `evm_network` - The target `EvmNetwork` for the nodes. + pub fn with_evm_network(mut self, evm_network: EvmNetwork) -> Self { + self.evm_network = evm_network; + self + } + + /// Sets the rewards address for the nodes. + /// + /// # Arguments + /// + /// * `rewards_address` - A valid `RewardsAddress` to collect rewards. + pub fn with_rewards_address(mut self, rewards_address: RewardsAddress) -> Self { + self.rewards_address = rewards_address; + self + } + + /// Configures the local mode for the network. + /// + /// # Arguments + /// + /// * `value` - If set to `true`, nodes will operate in local mode and listen only on `127.0.0.1`. + /// Otherwise, they listen on all interfaces (`0.0.0.0`). + pub fn with_local(mut self, value: bool) -> Self { + self.local = value; + self + } + + /// Enables or disables UPnP for the nodes. + /// + /// # Arguments + /// + /// * `value` - If `true`, nodes will attempt automatic port forwarding using UPnP. + pub fn with_upnp(mut self, value: bool) -> Self { + self.upnp = value; + self + } + + /// Sets the root directory for the nodes. + /// + /// # Arguments + /// + /// * `root_dir` - An optional file path where nodes will store their data. + pub fn with_root_dir(mut self, root_dir: Option) -> Self { + self.root_dir = root_dir; + self + } + + /// Specifies the number of nodes to spawn in the network. + /// + /// # Arguments + /// + /// * `size` - The number of nodes to create. Default is 5. + pub fn with_size(mut self, size: usize) -> Self { + self.size = size; + self + } + + /// Spawns the network with the configured parameters. + /// + /// # Returns + /// + /// A future resolving to a `SpawnedNetwork` containing the running nodes, + /// or an error if the spawning process fails. + pub async fn spawn(self) -> eyre::Result { + spawn_network( + self.evm_network, + self.rewards_address, + self.local, + self.upnp, + self.root_dir, + self.size, + ) + .await + } +} + +impl Default for NetworkSpawner { + fn default() -> Self { + Self::new() + } +} + +pub struct RunningNetwork { + running_nodes: Vec, +} + +impl RunningNetwork { + /// Returns a bootstrap peer from this network. + pub async fn bootstrap_peer(&self) -> Multiaddr { + self.running_nodes() + .first() + .expect("No nodes running, cannot get bootstrap peer") + .get_listen_addrs_with_peer_id() + .await + .expect("Could not get listen addresses for bootstrap peer") + .last() + .expect("Bootstrap peer has no listen addresses") + .clone() + } + + /// Return all running nodes. + pub fn running_nodes(&self) -> &Vec { + &self.running_nodes + } + + /// Shutdown all running nodes. + pub fn shutdown(self) { + for node in self.running_nodes.into_iter() { + node.shutdown(); + } + } +} + +async fn spawn_network( + evm_network: EvmNetwork, + rewards_address: RewardsAddress, + local: bool, + upnp: bool, + root_dir: Option, + size: usize, +) -> eyre::Result { + let mut running_nodes: Vec = vec![]; + + for i in 0..size { + let ip = match local { + true => IpAddr::V4(Ipv4Addr::LOCALHOST), + false => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + }; + + let socket_addr = SocketAddr::new(ip, 0); + + // Get the initial peers from the previously spawned nodes + let mut initial_peers: Vec = vec![]; + + for peer in running_nodes.iter() { + if let Ok(listen_addrs_with_peer_id) = peer.get_listen_addrs_with_peer_id().await { + initial_peers.extend(listen_addrs_with_peer_id); + } + } + + let node = NodeSpawner::new() + .with_socket_addr(socket_addr) + .with_evm_network(evm_network.clone()) + .with_rewards_address(rewards_address) + .with_initial_peers(initial_peers) + .with_local(local) + .with_upnp(upnp) + .with_root_dir(root_dir.clone()) + .spawn() + .await?; + + let listen_addrs = node.get_listen_addrs().await; + + info!( + "Spawned node #{} with listen addresses: {:?}", + i + 1, + listen_addrs + ); + + running_nodes.push(node); + } + + Ok(RunningNetwork { running_nodes }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::time::sleep; + + #[tokio::test(flavor = "multi_thread")] + async fn test_spawn_network() { + let network_size = 20; + + let running_network = NetworkSpawner::new() + .with_evm_network(Default::default()) + .with_local(true) + .with_size(network_size) + .spawn() + .await + .unwrap(); + + assert_eq!(running_network.running_nodes().len(), network_size); + + // Wait for nodes to dial each other + sleep(Duration::from_secs(10)).await; + + // Validate that all nodes know each other + for node in running_network.running_nodes() { + let known_peers = node.get_swarm_local_state().await.unwrap().connected_peers; + + assert_eq!(known_peers.len(), network_size - 1); + } + + running_network.shutdown(); + } +} diff --git a/ant-node/src/spawn/node_spawner.rs b/ant-node/src/spawn/node_spawner.rs new file mode 100644 index 0000000000..280c4f9b2a --- /dev/null +++ b/ant-node/src/spawn/node_spawner.rs @@ -0,0 +1,236 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::utils::get_root_dir_and_keypair; +use crate::{NodeBuilder, RunningNode}; +use ant_evm::{EvmNetwork, RewardsAddress}; +use libp2p::Multiaddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::PathBuf; + +pub struct NodeSpawner { + /// The socket address where the node will listen. + socket_addr: SocketAddr, + /// The EVM network the node will connect to. + evm_network: EvmNetwork, + /// The rewards address used for receiving rewards. + rewards_address: RewardsAddress, + /// A vector of `Multiaddr` representing the initial peers. + initial_peers: Vec, + /// A boolean indicating whether the node should run in local mode. + local: bool, + /// A boolean indicating whether UPnP should be enabled. + upnp: bool, + /// An optional `PathBuf` representing the root directory for the node. + root_dir: Option, +} + +impl NodeSpawner { + /// Create a new instance of `NodeSpawner` with default values. + pub fn new() -> Self { + Self { + socket_addr: SocketAddr::new(IpAddr::from(Ipv4Addr::UNSPECIFIED), 0), + evm_network: Default::default(), + rewards_address: Default::default(), + initial_peers: vec![], + local: false, + upnp: false, + root_dir: None, + } + } + + /// Set the socket address for the node. + /// + /// # Arguments + /// + /// * `socket_addr` - The `SocketAddr` where the node will listen. + pub fn with_socket_addr(mut self, socket_addr: SocketAddr) -> Self { + self.socket_addr = socket_addr; + self + } + + /// Set the EVM network for the node. + /// + /// # Arguments + /// + /// * `evm_network` - The `EvmNetwork` the node will connect to. + pub fn with_evm_network(mut self, evm_network: EvmNetwork) -> Self { + self.evm_network = evm_network; + self + } + + /// Set the rewards address for the node. + /// + /// # Arguments + /// + /// * `rewards_address` - The `RewardsAddress` used for distributing rewards. + pub fn with_rewards_address(mut self, rewards_address: RewardsAddress) -> Self { + self.rewards_address = rewards_address; + self + } + + /// Set the initial peers for the node. + /// + /// # Arguments + /// + /// * `initial_peers` - A vector of `Multiaddr` representing the initial peers. + pub fn with_initial_peers(mut self, initial_peers: Vec) -> Self { + self.initial_peers = initial_peers; + self + } + + /// Set the local mode flag for the node. + /// + /// # Arguments + /// + /// * `local` - A boolean indicating whether the node should run in local mode. + pub fn with_local(mut self, local: bool) -> Self { + self.local = local; + self + } + + /// Set the UPnP flag for the node. + /// + /// # Arguments + /// + /// * `upnp` - A boolean indicating whether UPnP should be enabled. + pub fn with_upnp(mut self, upnp: bool) -> Self { + self.upnp = upnp; + self + } + + /// Set the root directory for the node. + /// + /// # Arguments + /// + /// * `root_dir` - An optional `PathBuf` representing the root directory for the node. + pub fn with_root_dir(mut self, root_dir: Option) -> Self { + self.root_dir = root_dir; + self + } + + /// Spawn the node using the configured parameters. + /// + /// # Returns + /// + /// An `eyre::Result` containing a `RunningNode` if successful, or an error. + pub async fn spawn(self) -> eyre::Result { + spawn_node( + self.socket_addr, + self.evm_network, + self.rewards_address, + self.initial_peers, + self.local, + self.upnp, + &self.root_dir, + ) + .await + } +} + +impl Default for NodeSpawner { + fn default() -> Self { + Self::new() + } +} + +async fn spawn_node( + socket_addr: SocketAddr, + evm_network: EvmNetwork, + rewards_address: RewardsAddress, + initial_peers: Vec, + local: bool, + upnp: bool, + root_dir: &Option, +) -> eyre::Result { + let (root_dir, keypair) = get_root_dir_and_keypair(root_dir)?; + + let mut node_builder = NodeBuilder::new( + keypair, + rewards_address, + evm_network, + socket_addr, + local, + root_dir, + upnp, + ); + + if !initial_peers.is_empty() { + node_builder.initial_peers(initial_peers); + } + + let running_node = node_builder.build_and_run()?; + + // Verify that node is running + let mut retries: u8 = 0; + + let listen_addrs: Vec = loop { + // Wait till we have at least 1 listen addrs + if let Ok(listen_addrs) = running_node.get_listen_addrs().await { + if !listen_addrs.is_empty() { + break Ok(listen_addrs); + } + } + + if retries >= 3 { + break Err(eyre::eyre!( + "Failed to get listen addresses after {} retries", + retries + )); + } + + retries += 1; + + tokio::time::sleep(tokio::time::Duration::from_secs(retries as u64)).await; + }?; + + info!("Node listening on addresses: {:?}", listen_addrs); + + Ok(running_node) +} + +#[cfg(test)] +mod tests { + use super::*; + use ant_evm::EvmNetwork; + use futures::StreamExt; + use libp2p::swarm::dummy; + + #[tokio::test] + async fn test_launch_node() { + let evm_network = EvmNetwork::ArbitrumSepolia; + + let running_node = NodeSpawner::new() + .with_evm_network(evm_network) + .with_local(true) + .spawn() + .await + .unwrap(); + + let listen_addrs = running_node.get_listen_addrs().await.unwrap(); + + assert!(!listen_addrs.is_empty()); + + let mut swarm = libp2p::SwarmBuilder::with_new_identity() + .with_tokio() + .with_quic() + .with_behaviour(|_| dummy::Behaviour) + .unwrap() + .build(); + + let address = listen_addrs.first().unwrap().clone(); + + assert!(swarm.dial(address).is_ok()); + assert!(matches!( + swarm.next().await, + Some(libp2p::swarm::SwarmEvent::ConnectionEstablished { .. }) + )); + + running_node.shutdown(); + } +} diff --git a/ant-node/src/utils.rs b/ant-node/src/utils.rs new file mode 100644 index 0000000000..22377b36bd --- /dev/null +++ b/ant-node/src/utils.rs @@ -0,0 +1,88 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_protocol::node::get_antnode_root_dir; +use eyre::eyre; +use libp2p::identity::Keypair; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// The keypair is located inside the root directory. At the same time, when no dir is specified, +/// the dir name is derived from the keypair used in the application: the peer ID is used as the directory name. +pub fn get_root_dir_and_keypair(root_dir: &Option) -> eyre::Result<(PathBuf, Keypair)> { + match root_dir { + Some(dir) => { + std::fs::create_dir_all(dir)?; + + let secret_key_path = dir.join("secret-key"); + Ok((dir.clone(), keypair_from_path(secret_key_path)?)) + } + None => { + let secret_key = libp2p::identity::ed25519::SecretKey::generate(); + let keypair: Keypair = + libp2p::identity::ed25519::Keypair::from(secret_key.clone()).into(); + let peer_id = keypair.public().to_peer_id(); + + let dir = get_antnode_root_dir(peer_id)?; + std::fs::create_dir_all(&dir)?; + + let secret_key_path = dir.join("secret-key"); + + let mut file = create_secret_key_file(secret_key_path) + .map_err(|err| eyre!("could not create secret key file: {err}"))?; + file.write_all(secret_key.as_ref())?; + + Ok((dir, keypair)) + } + } +} + +fn keypair_from_path(path: impl AsRef) -> eyre::Result { + let keypair = match std::fs::read(&path) { + // If the file is opened successfully, read the key from it + Ok(key) => { + let keypair = Keypair::ed25519_from_bytes(key) + .map_err(|err| eyre!("could not read ed25519 key from file: {err}"))?; + + info!("loaded secret key from file: {:?}", path.as_ref()); + + keypair + } + // In case the file is not found, generate a new keypair and write it to the file + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let secret_key = libp2p::identity::ed25519::SecretKey::generate(); + let mut file = create_secret_key_file(&path) + .map_err(|err| eyre!("could not create secret key file: {err}"))?; + file.write_all(secret_key.as_ref())?; + + info!("generated new key and stored to file: {:?}", path.as_ref()); + + libp2p::identity::ed25519::Keypair::from(secret_key).into() + } + // Else the file can't be opened, for whatever reason (e.g. permissions). + Err(err) => { + return Err(eyre!("failed to read secret key file: {err}")); + } + }; + + Ok(keypair) +} + +fn create_secret_key_file(path: impl AsRef) -> eyre::Result { + let mut opt = std::fs::OpenOptions::new(); + let _ = opt.write(true).create_new(true); + + // On Unix systems, make sure only the current user can read/write. + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let _ = opt.mode(0o600); + } + + opt.open(path) +} diff --git a/ant-node/tests/common/client.rs b/ant-node/tests/common/client.rs index faf8c1ae05..851edb53de 100644 --- a/ant-node/tests/common/client.rs +++ b/ant-node/tests/common/client.rs @@ -52,7 +52,7 @@ pub async fn get_client_and_funded_wallet() -> (Client, Wallet) { } /// Get the node count -/// If SN_INVENTORY flag is passed, the node count is obtained from the the droplet +/// If SN_INVENTORY flag is passed, the node count is obtained from the droplet /// else return the local node count pub fn get_node_count() -> usize { match DeploymentInventory::load() { diff --git a/ant-node/tests/data_with_churn.rs b/ant-node/tests/data_with_churn.rs index 87261779c4..9e6871df99 100644 --- a/ant-node/tests/data_with_churn.rs +++ b/ant-node/tests/data_with_churn.rs @@ -6,9 +6,6 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -// TODO: Remove this once the registers are removed -#![expect(deprecated)] - mod common; use crate::common::{ @@ -16,14 +13,19 @@ use crate::common::{ NodeRestart, }; use ant_logging::LogBuilder; -use ant_protocol::{storage::ChunkAddress, NetworkAddress}; +use ant_protocol::{ + storage::{ChunkAddress, GraphEntry, GraphEntryAddress, PointerTarget, ScratchpadAddress}, + NetworkAddress, +}; use autonomi::{Client, Wallet}; +use bls::{PublicKey, SecretKey}; +use bytes::Bytes; use common::client::transfer_to_new_wallet; use eyre::{bail, ErrReport, Result}; use rand::Rng; use self_encryption::MAX_CHUNK_SIZE; use std::{ - collections::{BTreeMap, VecDeque}, + collections::{BTreeMap, HashMap, VecDeque}, fmt, fs::create_dir_all, sync::{Arc, LazyLock}, @@ -39,8 +41,10 @@ const TOKENS_TO_TRANSFER: usize = 10000000; const EXTRA_CHURN_COUNT: u32 = 5; const CHURN_CYCLES: u32 = 2; -const CHUNK_CREATION_RATIO_TO_CHURN: u32 = 15; -const REGISTER_CREATION_RATIO_TO_CHURN: u32 = 15; +const CHUNK_CREATION_RATIO_TO_CHURN: u32 = 13; +const POINTER_CREATION_RATIO_TO_CHURN: u32 = 11; +const SCRATCHPAD_CREATION_RATIO_TO_CHURN: u32 = 9; +const GRAPHENTRY_CREATION_RATIO_TO_CHURN: u32 = 7; static DATA_SIZE: LazyLock = LazyLock::new(|| *MAX_CHUNK_SIZE / 3); @@ -101,7 +105,7 @@ async fn data_availability_during_churn() -> Result<()> { // Create a cross thread usize for tracking churned nodes let churn_count = Arc::new(RwLock::new(0_usize)); - // Allow to disable Registers data creation/checks, storing and querying only Chunks during churn. + // Allow to disable non-chunk data_types creation/checks. // Default to be not carry out chunks only during churn. let chunks_only = std::env::var("CHUNKS_ONLY").is_ok(); @@ -124,21 +128,6 @@ async fn data_availability_during_churn() -> Result<()> { // Shared bucket where we keep track of content created/stored on the network let content = ContentList::default(); - // Spawn a task to create Registers at random locations, - // at a higher frequency than the churning events - let create_register_handle = if !chunks_only { - let register_wallet = transfer_to_new_wallet(&main_wallet, TOKENS_TO_TRANSFER).await?; - let create_register_handle = create_registers_task( - client.clone(), - register_wallet, - Arc::clone(&content), - churn_period, - ); - Some(create_register_handle) - } else { - None - }; - println!("Uploading some chunks before carry out node churning"); info!("Uploading some chunks before carry out node churning"); @@ -151,6 +140,51 @@ async fn data_availability_during_churn() -> Result<()> { churn_period, ); + // Spawn a task to create Pointers at random locations, + // at a higher frequency than the churning events + let create_pointer_handle = if !chunks_only { + let pointer_wallet = transfer_to_new_wallet(&main_wallet, TOKENS_TO_TRANSFER).await?; + let create_pointer_handle = create_pointers_task( + client.clone(), + pointer_wallet, + Arc::clone(&content), + churn_period, + ); + Some(create_pointer_handle) + } else { + None + }; + + // Spawn a task to create GraphEntry at random locations, + // at a higher frequency than the churning events + let create_graph_entry_handle = if !chunks_only { + let graph_entry_wallet = transfer_to_new_wallet(&main_wallet, TOKENS_TO_TRANSFER).await?; + let create_graph_entry_handle = create_graph_entry_task( + client.clone(), + graph_entry_wallet, + Arc::clone(&content), + churn_period, + ); + Some(create_graph_entry_handle) + } else { + None + }; + + // Spawn a task to create ScratchPad at random locations, + // at a higher frequency than the churning events + let create_scratchpad_handle = if !chunks_only { + let scratchpad_wallet = transfer_to_new_wallet(&main_wallet, TOKENS_TO_TRANSFER).await?; + let create_scratchpad_handle = create_scratchpad_task( + client.clone(), + scratchpad_wallet, + Arc::clone(&content), + churn_period, + ); + Some(create_scratchpad_handle) + } else { + None + }; + // Spawn a task to churn nodes churn_nodes_task(Arc::clone(&churn_count), test_duration, churn_period); @@ -187,9 +221,19 @@ async fn data_availability_during_churn() -> Result<()> { if store_chunks_handle.is_finished() { bail!("Store chunks task has finished before the test duration. Probably due to an error."); } - if let Some(handle) = &create_register_handle { + if let Some(handle) = &create_pointer_handle { if handle.is_finished() { - bail!("Create registers task has finished before the test duration. Probably due to an error."); + bail!("Create Pointers task has finished before the test duration. Probably due to an error."); + } + } + if let Some(handle) = &create_graph_entry_handle { + if handle.is_finished() { + bail!("Create GraphEntry task has finished before the test duration. Probably due to an error."); + } + } + if let Some(handle) = &create_scratchpad_handle { + if handle.is_finished() { + bail!("Create ScratchPad task has finished before the test duration. Probably due to an error."); } } @@ -267,55 +311,342 @@ async fn data_availability_during_churn() -> Result<()> { Ok(()) } -// Spawns a task which periodically creates Registers at random locations. -fn create_registers_task( +// Spawns a task which periodically creates ScratchPads at random locations. +fn create_scratchpad_task( client: Client, wallet: Wallet, content: ContentList, churn_period: Duration, ) -> JoinHandle> { let handle: JoinHandle> = tokio::spawn(async move { - // Create Registers at a higher frequency than the churning events - let delay = churn_period / REGISTER_CREATION_RATIO_TO_CHURN; + // Map of the ownership, allowing the later on update can be undertaken. + let mut owners: HashMap = HashMap::new(); + + // Create ScratchPad at a higher frequency than the churning events + let delay = churn_period / SCRATCHPAD_CREATION_RATIO_TO_CHURN; loop { - let owner = Client::register_generate_key(); - let random_name = XorName(rand::random()).to_string(); - let random_data = gen_random_data(*DATA_SIZE); + sleep(delay).await; + + // 50% chance to carry out update instead of creation. + let is_update: bool = if owners.is_empty() { + false + } else { + rand::random() + }; + + let content_type: u64 = rand::random(); + let data_byte: u8 = rand::random(); + let mut data = vec![data_byte; 100]; + rand::thread_rng().fill(&mut data[..]); + let bytes = Bytes::from(data); + + let mut retries = 1; + if is_update { + let index = rand::thread_rng().gen_range(0..owners.len()); + let iterator: Vec<_> = owners.iter().collect(); + let (addr, owner) = iterator[index]; + + loop { + match client.scratchpad_update(owner, content_type, &bytes).await { + Ok(_) => { + println!("Updated ScratchPad at {addr:?} after a delay of: {delay:?}"); + break; + } + Err(err) => { + println!("Failed to update ScratchPad at {addr:?}. Retrying ..."); + error!("Failed to update ScratchPad at {addr:?}. Retrying ..."); + if retries >= 3 { + println!( + "Failed to update pointer at {addr:?} after 3 retries: {err}" + ); + error!( + "Failed to update pointer at {addr:?} after 3 retries: {err}" + ); + bail!( + "Failed to update pointer at {addr:?} after 3 retries: {err}" + ); + } + retries += 1; + } + } + } + } else { + let owner = SecretKey::random(); + loop { + match client + .scratchpad_create(&owner, content_type, &bytes, (&wallet).into()) + .await + { + Ok((cost, addr)) => { + println!("Created new ScratchPad at {addr:?} with cost of {cost:?} after a delay of: {delay:?}"); + let net_addr = NetworkAddress::ScratchpadAddress(addr); + content.write().await.push_back(net_addr); + let _ = owners.insert(addr, owner); + break; + } + Err(err) => { + println!("Failed to create ScratchPad: {err:?}. Retrying ..."); + error!("Failed to create ScratchPad: {err:?}. Retrying ..."); + if retries >= 3 { + println!("Failed to create ScratchPad after 3 retries: {err}"); + error!("Failed to create ScratchPad after 3 retries: {err}"); + bail!("Failed to create ScratchPad after 3 retries: {err}"); + } + retries += 1; + } + } + } + } + } + }); + handle +} +// Spawns a task which periodically creates GraphEntry at random locations. +fn create_graph_entry_task( + client: Client, + wallet: Wallet, + content_list: ContentList, + churn_period: Duration, +) -> JoinHandle> { + let handle: JoinHandle> = tokio::spawn(async move { + // Map of the ownership, allowing the later on update can be undertaken. + // In this test scenario, we only test simple GraphEntry tree structure: 1-parent-1-output + // The tree structure is stored as a vector (last one being the latest) + let mut growing_history: Vec> = vec![]; + let mut owners: HashMap = HashMap::new(); + + // Create GraphEntry at a higher frequency than the churning events + let delay = churn_period / GRAPHENTRY_CREATION_RATIO_TO_CHURN; + + loop { sleep(delay).await; + // 50% chance of `growing` (i.e. has existing one as partent) instead of creation new. + let is_growing: bool = if growing_history.is_empty() { + false + } else { + rand::random() + }; + + let output = SecretKey::random(); + let output_content: [u8; 32] = rand::random(); + let outputs = vec![(output.public_key(), output_content)]; + + #[allow(unused_assignments)] + let mut index = growing_history.len(); + let mut graph_entry_to_put = None; + if is_growing { + index = rand::thread_rng().gen_range(0..growing_history.len()); + let Some(addr) = growing_history[index].last() else { + println!("Doesn't have history GraphEntry of {index:?}"); + error!("Doesn't have history GraphEntry of {index:?}"); + continue; + }; + + let mut retries = 1; + loop { + match client.graph_entry_get(addr).await { + Ok(graph_entry) => { + println!("Fetched graph_entry at {addr:?}"); + + let Some((old_output, old_content)) = graph_entry.descendants.last() + else { + println!("Can't get output from the graph_entry of {addr:?}"); + error!("Can't get output from the graph_entry of {addr:?}"); + break; + }; + + // The previous output now becomes the owner. + let Some(owner) = owners.get(old_output) else { + println!("Can't get secret_key of {output:?}"); + error!("Can't get secret_key of {output:?}"); + break; + }; + + let parents = vec![graph_entry.owner]; + let graph_entry = + GraphEntry::new(owner, parents, *old_content, outputs); + + growing_history[index].push(graph_entry.address()); + + graph_entry_to_put = Some(graph_entry); + break; + } + Err(err) => { + println!( + "Failed to get graph_entry at {addr:?}: {err:?}. Retrying ..." + ); + error!("Failed to get graph_entry at {addr:?} : {err:?}. Retrying ..."); + if retries >= 3 { + println!( + "Failed to get graph_entry at {addr:?} after 3 retries: {err}" + ); + error!( + "Failed to get graph_entry at {addr:?} after 3 retries: {err}" + ); + bail!( + "Failed to get graph_entry at {addr:?} after 3 retries: {err}" + ); + } + retries += 1; + sleep(delay).await; + } + } + } + } else { + let owner = SecretKey::random(); + let content: [u8; 32] = rand::random(); + let parents = vec![]; + let graph_entry = GraphEntry::new(&owner, parents, content, outputs); + + growing_history.push(vec![graph_entry.address()]); + let _ = owners.insert(owner.public_key(), owner); + + graph_entry_to_put = Some(graph_entry); + }; + + let Some(graph_entry) = graph_entry_to_put else { + println!("Doesn't have graph_entry to put to network."); + error!("Doesn't have graph_entry to put to network."); + continue; + }; + + let _ = owners.insert(output.public_key(), output); + let mut retries = 1; loop { match client - .register_create( - Some(random_data.clone()), - &random_name, - owner.clone(), - &wallet, - ) + .graph_entry_put(graph_entry.clone(), (&wallet).into()) .await { - Ok(register) => { - let addr = register.address(); - println!("Created new Register ({addr:?}) after a delay of: {delay:?}"); - content - .write() - .await - .push_back(NetworkAddress::RegisterAddress(*addr)); + Ok((cost, addr)) => { + println!("Uploaded graph_entry to {addr:?} with cost of {cost:?} after a delay of: {delay:?}"); + let net_addr = NetworkAddress::GraphEntryAddress(addr); + content_list.write().await.push_back(net_addr); break; } Err(err) => { - println!("Failed to create register: {err:?}. Retrying ..."); - error!("Failed to create register: {err:?}. Retrying ..."); + println!("Failed to upload graph_entry: {err:?}. Retrying ..."); + error!("Failed to upload graph_entry: {err:?}. Retrying ..."); if retries >= 3 { - println!("Failed to create register after 3 retries: {err}"); - error!("Failed to create register after 3 retries: {err}"); - bail!("Failed to create register after 3 retries: {err}"); + println!("Failed to upload graph_entry after 3 retries: {err}"); + error!("Failed to upload graph_entry after 3 retries: {err}"); + bail!("Failed to upload graph_entry after 3 retries: {err}"); } retries += 1; + sleep(delay).await; + } + } + } + } + }); + handle +} + +// Spawns a task which periodically creates Pointers at random locations. +fn create_pointers_task( + client: Client, + wallet: Wallet, + content: ContentList, + churn_period: Duration, +) -> JoinHandle> { + let handle: JoinHandle> = tokio::spawn(async move { + // Map of the ownership, allowing the later on update can be undertaken. + let mut owners: HashMap = HashMap::new(); + + // Create Pointers at a higher frequency than the churning events + let delay = churn_period / POINTER_CREATION_RATIO_TO_CHURN; + + loop { + sleep(delay).await; + + #[allow(unused_assignments)] + let mut pointer_addr = None; + + // 50% chance to carry out update instead of creation. + let is_update: bool = if owners.is_empty() { + false + } else { + rand::random() + }; + + let mut retries = 1; + + if is_update { + let index = rand::thread_rng().gen_range(0..owners.len()); + let iterator: Vec<_> = owners.iter().collect(); + let (addr, (owner, old_target)) = iterator[index]; + + let new_target = + PointerTarget::ChunkAddress(ChunkAddress::new(XorName(rand::random()))); + loop { + match client.pointer_update(owner, new_target.clone()).await { + Ok(_) => { + println!("Updated Pointer at {addr:?} with {old_target:?} to new target {new_target:?} after a delay of: {delay:?}"); + pointer_addr = Some((addr.clone(), None, new_target)); + break; + } + Err(err) => { + println!( + "Failed to update pointer at {addr:?} with {old_target:?}: {err:?}. Retrying ..." + ); + error!( + "Failed to update pointer at {addr:?} with {old_target:?}: {err:?}. Retrying ..." + ); + if retries >= 3 { + println!("Failed to update pointer at {addr:?} with {old_target:?} after 3 retries: {err}"); + error!("Failed to update pointer at {addr:?} with {old_target:?} after 3 retries: {err}"); + bail!("Failed to update pointer at {addr:?} with {old_target:?} after 3 retries: {err}"); + } + retries += 1; + } } } + } else { + let owner = SecretKey::random(); + let pointer_target = + PointerTarget::ChunkAddress(ChunkAddress::new(XorName(rand::random()))); + loop { + match client + .pointer_create(&owner, pointer_target.clone(), (&wallet).into()) + .await + { + Ok((cost, addr)) => { + println!("Created new Pointer ({pointer_target:?}) at {addr:?} with cost of {cost:?} after a delay of: {delay:?}"); + let net_addr = NetworkAddress::PointerAddress(addr); + pointer_addr = Some((net_addr.clone(), Some(owner), pointer_target)); + content.write().await.push_back(net_addr); + break; + } + Err(err) => { + println!( + "Failed to create pointer {pointer_target:?}: {err:?}. Retrying ..." + ); + error!( + "Failed to create pointer {pointer_target:?}: {err:?}. Retrying ..." + ); + if retries >= 3 { + println!("Failed to create pointer {pointer_target:?} after 3 retries: {err}"); + error!("Failed to create pointer {pointer_target:?} after 3 retries: {err}"); + bail!("Failed to create pointer {pointer_target:?} after 3 retries: {err}"); + } + retries += 1; + } + } + } + } + match pointer_addr { + Some((addr, Some(owner), target)) => { + let _ = owners.insert(addr, (owner, target)); + } + Some((addr, None, new_target)) => { + if let Some((_owner, target)) = owners.get_mut(&addr) { + *target = new_target; + } + } + _ => {} } } }); @@ -352,7 +683,7 @@ fn store_chunks_task( println!("Error to put chunk: {err:?}"); error!("Error to put chunk: {err:?}") }) { - Ok(data_map) => { + Ok((_cost, data_map)) => { println!("Stored Chunk/s at {data_map:?} after a delay of: {delay:?}"); info!("Stored Chunk/s at {data_map:?} after a delay of: {delay:?}"); @@ -526,7 +857,7 @@ async fn final_retry_query_content( } else { attempts += 1; let delay = 2 * churn_period; - debug!("Delaying last check for {delay:?} ..."); + debug!("Delaying last check of {net_addr:?} for {delay:?} ..."); sleep(delay).await; continue; } @@ -540,14 +871,23 @@ async fn final_retry_query_content( async fn query_content(client: &Client, net_addr: &NetworkAddress) -> Result<()> { match net_addr { - NetworkAddress::RegisterAddress(addr) => { - let _ = client.register_get(*addr).await?; + NetworkAddress::ChunkAddress(addr) => { + client.data_get_public(addr.xorname()).await?; Ok(()) } - NetworkAddress::ChunkAddress(addr) => { - client.data_get_public(*addr.xorname()).await?; + NetworkAddress::PointerAddress(addr) => { + let _ = client.pointer_get(addr).await?; + Ok(()) + } + NetworkAddress::GraphEntryAddress(addr) => { + let _ = client.graph_entry_get(addr).await?; + Ok(()) + } + NetworkAddress::ScratchpadAddress(addr) => { + let _ = client.scratchpad_get(addr).await?; Ok(()) } - _other => Ok(()), // we don't create/store any other type of content in this test yet + // Drain the enum to ensure all native supported data_types are covered + NetworkAddress::PeerId(_) | NetworkAddress::RecordKey(_) => Ok(()), } } diff --git a/ant-node/tests/storage_payments.rs b/ant-node/tests/storage_payments.rs index bfb6d4ae75..499228c4bb 100644 --- a/ant-node/tests/storage_payments.rs +++ b/ant-node/tests/storage_payments.rs @@ -19,10 +19,9 @@ // use ant_networking::{GetRecordError, NetworkError}; // use ant_protocol::{ // error::Error as ProtocolError, -// storage::{ChunkAddress, RegisterAddress}, +// storage::ChunkAddress, // NetworkAddress, // }; -// use ant_registers::Permissions; // use std::collections::BTreeMap; // use tokio::time::{sleep, Duration}; // use tracing::info; @@ -262,164 +261,3 @@ // Ok(()) // } - -// #[tokio::test] -// async fn storage_payment_register_creation_succeeds() -> Result<()> { -// let _log_guards = LogBuilder::init_single_threaded_tokio_test("storage_payments", true); - -// let paying_wallet_dir = TempDir::new()?; - -// let (client, paying_wallet) = get_client_and_funded_wallet(paying_wallet_dir.path()).await?; -// let mut wallet_client = WalletClient::new(client.clone(), paying_wallet); - -// let mut rng = rand::thread_rng(); -// let xor_name = XorName::random(&mut rng); -// let address = RegisterAddress::new(xor_name, client.signer_pk()); -// let net_addr = NetworkAddress::from_register_address(address); -// info!("Paying for random Register address {net_addr:?} ..."); - -// let _cost = wallet_client -// .pay_for_storage(std::iter::once(net_addr)) -// .await?; - -// let (mut register, _cost, _royalties_fees) = client -// .create_and_pay_for_register(xor_name, &mut wallet_client, true, Permissions::default()) -// .await?; - -// println!("Newly created register has {} ops", register.read().len()); - -// let retrieved_reg = client.get_register(address).await?; - -// assert_eq!(register.read(), retrieved_reg.read()); - -// let random_entry = rng.gen::<[u8; 32]>().to_vec(); - -// register.write(&random_entry)?; - -// println!( -// "Register has {} ops after first write", -// register.read().len() -// ); - -// register.sync(&mut wallet_client, true, None).await?; - -// let retrieved_reg = client.get_register(address).await?; - -// assert_eq!(retrieved_reg.read().iter().next().unwrap().1, random_entry); - -// assert_eq!(retrieved_reg.read().len(), 1); - -// for index in 1..10 { -// println!("current index is {index}"); -// let random_entry = rng.gen::<[u8; 32]>().to_vec(); - -// register.write(&random_entry)?; -// register.sync(&mut wallet_client, true, None).await?; - -// let retrieved_reg = client.get_register(address).await?; - -// println!( -// "current retrieved register entry length is {}", -// retrieved_reg.read().len() -// ); -// println!("current expected entry length is {}", register.read().len()); - -// println!( -// "current retrieved register ops length is {}", -// retrieved_reg.ops.len() -// ); -// println!("current local cached ops length is {}", register.ops.len()); - -// assert_eq!(retrieved_reg.read().len(), register.read().len()); - -// assert_eq!(retrieved_reg.read().iter().next().unwrap().1, random_entry); - -// println!("Current fetched register is {:?}", retrieved_reg.register); -// println!( -// "Fetched register has update history of {}", -// retrieved_reg.register.log_update_history() -// ); - -// std::thread::sleep(std::time::Duration::from_millis(1000)); -// } - -// Ok(()) -// } - -// #[tokio::test] -// #[ignore = "Test currently invalid as we always try to pay and upload registers if none found... need to check if this test is valid"] -// async fn storage_payment_register_creation_and_mutation_fails() -> Result<()> { -// let _log_guards = LogBuilder::init_single_threaded_tokio_test("storage_payments", true); - -// let paying_wallet_dir = TempDir::new()?; - -// let (client, paying_wallet) = get_client_and_funded_wallet(paying_wallet_dir.path()).await?; -// let mut wallet_client = WalletClient::new(client.clone(), paying_wallet); - -// let mut rng = rand::thread_rng(); -// let xor_name = XorName::random(&mut rng); -// let address = RegisterAddress::new(xor_name, client.signer_pk()); -// let net_address = -// NetworkAddress::RegisterAddress(RegisterAddress::new(xor_name, client.signer_pk())); - -// let mut no_data_payments = BTreeMap::default(); -// no_data_payments.insert( -// net_address -// .as_xorname() -// .expect("RegisterAddress should convert to XorName"), -// ( -// ant_evm::utils::dummy_address(), -// PaymentQuote::test_dummy(xor_name, AttoTokens::from_u64(0)), -// vec![], -// ), -// ); - -// println!( -// "current retrieved register entry length is {}", -// retrieved_reg.read().len() -// ); -// println!("current expected entry length is {}", register.read().len()); - -// println!( -// "current retrieved register ops length is {}", -// retrieved_reg.ops_list().len() -// ); -// println!( -// "current local cached ops length is {}", -// register.ops_list().len() -// ); - -// // TODO adapt to evm -// // let _ = wallet_client -// // .mut_wallet() -// // .send_storage_payment(&no_data_payments) -// // .await?; - -// // this should fail to store as the amount paid is not enough -// let (mut register, _cost, _royalties_fees) = client -// .create_and_pay_for_register(xor_name, &mut wallet_client, false, Permissions::default()) -// .await?; - -// sleep(Duration::from_secs(5)).await; -// assert!(matches!( -// client.get_register(address).await, -// Err(ClientError::Protocol(ProtocolError::RegisterNotFound(addr))) if *addr == address -// )); - -// println!("Current fetched register is {:?}", retrieved_reg.address()); -// println!( -// "Fetched register has update history of {}", -// retrieved_reg.log_update_history() -// ); - -// let random_entry = rng.gen::<[u8; 32]>().to_vec(); -// register.write(&random_entry)?; - -// sleep(Duration::from_secs(5)).await; -// assert!(matches!( -// register.sync(&mut wallet_client, false, None).await, -// Err(ClientError::Protocol(ProtocolError::RegisterNotFound(addr))) if *addr == address -// )); - -// Ok(()) -// } diff --git a/ant-node/tests/verify_data_location.rs b/ant-node/tests/verify_data_location.rs index e8e2c6938a..9aad7b690c 100644 --- a/ant-node/tests/verify_data_location.rs +++ b/ant-node/tests/verify_data_location.rs @@ -6,14 +6,10 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -// TODO: Remove this once the registers are removed -#![expect(deprecated)] -#![allow(clippy::mutable_key_type)] - mod common; use ant_logging::LogBuilder; -use ant_networking::{sleep, sort_peers_by_key}; +use ant_networking::sort_peers_by_key; use ant_protocol::{ antnode_proto::{NodeInfoRequest, RecordAddressesRequest}, NetworkAddress, PrettyPrintRecordKey, CLOSE_GROUP_SIZE, @@ -62,9 +58,6 @@ const CHURN_COUNT: u8 = 20; /// Default number of chunks that should be PUT to the network. /// It can be overridden by setting the 'CHUNK_COUNT' env var. const CHUNK_COUNT: usize = 5; -/// Default number of registers that should be PUT to the network. -/// It can be overridden by setting the 'REGISTER_COUNT' env var. -const REGISTER_COUNT: usize = 5; type NodeIndex = usize; type RecordHolders = HashMap>; @@ -84,17 +77,12 @@ async fn verify_data_location() -> Result<()> { } else { CHUNK_COUNT }; - let register_count = if let Ok(str) = std::env::var("REGISTER_COUNT") { - str.parse::()? - } else { - REGISTER_COUNT - }; println!( - "Performing data location verification with a churn count of {churn_count} and n_chunks {chunk_count}, n_registers {register_count}\nIt will take approx {:?}", + "Performing data location verification with a churn count of {churn_count} and n_chunks {chunk_count}\nIt will take approx {:?}", VERIFICATION_DELAY*churn_count as u32 ); info!( - "Performing data location verification with a churn count of {churn_count} and n_chunks {chunk_count}, n_registers {register_count}\nIt will take approx {:?}", + "Performing data location verification with a churn count of {churn_count} and n_chunks {chunk_count}\nIt will take approx {:?}", VERIFICATION_DELAY*churn_count as u32 ); let node_rpc_address = get_all_rpc_addresses(true)?; @@ -103,7 +91,6 @@ async fn verify_data_location() -> Result<()> { let (client, wallet) = get_client_and_funded_wallet().await; store_chunks(&client, chunk_count, &wallet).await?; - store_registers(&client, register_count, &wallet).await?; // Verify data location initially verify_location(&all_peers, &node_rpc_address).await?; @@ -376,61 +363,3 @@ async fn store_chunks( Ok(()) } - -async fn store_registers( - client: &Client, - register_count: usize, - wallet: &evmlib::wallet::Wallet, -) -> Result<()> { - let start = Instant::now(); - - let mut uploaded_registers_count = 0; - loop { - if uploaded_registers_count >= register_count { - break; - } - // Owner key of the register. - let key = bls::SecretKey::random(); - - // Create a register with the value [1, 2, 3, 4] - let rand_name: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(10) - .map(char::from) - .collect(); - let register = client - .register_create( - Some(vec![1, 2, 3, 4].into()), - &rand_name, - key.clone(), - wallet, - ) - .await?; - - println!("Created Register at {:?}", register.address()); - debug!("Created Register at {:?}", register.address()); - sleep(Duration::from_secs(5)).await; - - // Update the register with the value [5, 6, 7, 8] - client - .register_update(register.clone(), vec![5, 6, 7, 8].into(), key) - .await?; - - println!("Updated Register at {:?}", register.address()); - debug!("Updated Register at {:?}", register.address()); - - uploaded_registers_count += 1; - } - println!( - "{register_count:?} Registers were stored in {:?}", - start.elapsed() - ); - info!( - "{register_count:?} Registers were stored in {:?}", - start.elapsed() - ); - - // to make sure the last register was stored - sleep(Duration::from_secs(10)).await; - Ok(()) -} diff --git a/ant-protocol/Cargo.toml b/ant-protocol/Cargo.toml index cdc866bdc9..dd057c72d4 100644 --- a/ant-protocol/Cargo.toml +++ b/ant-protocol/Cargo.toml @@ -7,30 +7,27 @@ license = "GPL-3.0" name = "ant-protocol" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.3.3" +version = "1.0.0" [features] default = [] rpc = ["tonic", "prost"] [dependencies] -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-registers = { path = "../ant-registers", version = "0.4.7" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } -color-eyre = "0.6.2" +color-eyre = "0.6.3" crdts = { version = "7.3", default-features = false, features = ["merkle"] } custom_debug = "~0.6.1" dirs-next = "~2.0.0" -exponential-backoff = "2.0.0" hex = "~0.4.3" lazy_static = "1.4.0" -libp2p = { version = "0.54.1", features = ["identify", "kad"] } -# # watch out updating this, protoc compiler needs to be installed on all build systems -# # arm builds + musl are very problematic -# prost and tonic are needed for the RPC server messages, not the underlying protocol +libp2p = { version = "0.55.0", features = ["identify", "kad"] } +prometheus-client = { version = "0.22" } prost = { version = "0.9", optional = true } +rand = "0.8" rmp-serde = "1.1.1" serde = { version = "1.0.133", features = ["derive", "rc"] } serde_json = "1.0" @@ -38,13 +35,18 @@ sha2 = "0.10.7" thiserror = "1.0.23" tiny-keccak = { version = "~2.0.2", features = ["sha3"] } tracing = { version = "~0.1.26" } -tonic = { version = "0.6.2", optional = true, default-features = false, features = ["prost", "tls", "codegen"] } +tonic = { version = "0.6.2", optional = true, default-features = false, features = [ + "prost", + "tls", + "codegen", +] } xor_name = "5.0.0" [build-dependencies] -# watch out updating this, protoc compiler needs to be installed on all build systems -# arm builds + musl are very problematic tonic-build = { version = "~0.6.2" } [lints] workspace = true + +[dev-dependencies] +rand = "0.8" diff --git a/ant-protocol/README.md b/ant-protocol/README.md index 3239f0f2be..5463d2532e 100644 --- a/ant-protocol/README.md +++ b/ant-protocol/README.md @@ -25,8 +25,6 @@ The `error.rs` file contains the definitions for various errors that can occur w - Example: `Result::Err(Error::ChunkNotFound(chunk_address))` - `ChunkNotStored(XorName)`: Indicates that a chunk was not stored. - Example: `Result::Err(Error::ChunkNotStored(xor_name))` -- `RegisterNotFound(Box)`: Indicates that a register was not found. - - Example: `Result::Err(Error::RegisterNotFound(register_address))` ## Messages @@ -71,7 +69,7 @@ The `storage` module handles the storage aspects of the protocol. ### API Calls - `ChunkAddress`: Address of a chunk in the network. -- `TransactionAddress`: Address of a Transaction in the network. +- `GraphEntryAddress`: Address of a Transaction in the network. - `Header`: Header information for storage items. ## Protobuf Definitions diff --git a/ant-protocol/src/error.rs b/ant-protocol/src/error.rs index bc784860e1..e5cde6ba9f 100644 --- a/ant-protocol/src/error.rs +++ b/ant-protocol/src/error.rs @@ -6,7 +6,8 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::{storage::RegisterAddress, NetworkAddress, PrettyPrintRecordKey}; +use crate::{NetworkAddress, PrettyPrintRecordKey}; +use libp2p::kad::store; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -14,7 +15,7 @@ use thiserror::Error; pub type Result = std::result::Result; /// Main error types for the SAFE protocol. -#[derive(Error, Clone, PartialEq, Eq, Serialize, Deserialize, custom_debug::Debug)] +#[derive(Error, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Error { // ---------- Misc errors @@ -31,21 +32,8 @@ pub enum Error { #[error("Chunk does not exist {0:?}")] ChunkDoesNotExist(NetworkAddress), - // ---------- Register Errors - #[error("Register not found: {0}")] - RegisterNotFound(Box), - #[error("The Register was already created by another owner: {0:?}")] - RegisterAlreadyClaimed(bls::PublicKey), - #[error("Peer {holder:?} cannot find Record {key:?}")] - RegisterRecordNotFound { - /// Holder that being contacted - holder: Box, - /// Key of the missing record - key: Box, - }, - // ---------- Scratchpad errors - /// The provided String can't be deserialized as a RegisterAddress + /// The provided String can't be deserialized as a ScratchpadAddress #[error("Failed to deserialize hex ScratchpadAddress")] ScratchpadHexDeserializeFailed, /// The provided SecretyKey failed to decrypt the data @@ -82,3 +70,15 @@ pub enum Error { #[error("The record already exists, so do not charge for it: {0:?}")] RecordExists(PrettyPrintRecordKey<'static>), } + +impl From for store::Error { + fn from(_err: Error) -> Self { + store::Error::ValueTooLarge + } +} + +impl From for Error { + fn from(_err: store::Error) -> Self { + Error::RecordParsingFailed + } +} diff --git a/ant-protocol/src/lib.rs b/ant-protocol/src/lib.rs index 936d474246..0846846fa8 100644 --- a/ant-protocol/src/lib.rs +++ b/ant-protocol/src/lib.rs @@ -17,7 +17,7 @@ pub mod messages; pub mod node; /// RPC commands to node pub mod node_rpc; -/// Storage types for transactions, chunks and registers. +/// Storage types for GraphEntry and Chunk pub mod storage; /// Network versioning pub mod version; @@ -29,26 +29,24 @@ pub mod antnode_proto { tonic::include_proto!("antnode_proto"); } pub use error::Error; +pub use error::Error as NetworkError; use storage::ScratchpadAddress; -use self::storage::{ChunkAddress, RegisterAddress, TransactionAddress}; +use self::storage::{ChunkAddress, GraphEntryAddress, PointerAddress}; /// Re-export of Bytes used throughout the protocol pub use bytes::Bytes; -use ant_evm::U256; use libp2p::{ kad::{KBucketDistance as Distance, KBucketKey as Key, RecordKey}, multiaddr::Protocol, Multiaddr, PeerId, }; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::str::FromStr; use std::{ borrow::Cow, fmt::{self, Debug, Display, Formatter, Write}, }; -use xor_name::XorName; /// The maximum number of peers to return in a `GetClosestPeers` response. /// This is the group size used in safe network protocol to be responsible for @@ -68,18 +66,6 @@ pub fn get_port_from_multiaddr(multi_addr: &Multiaddr) -> Option { None } -// This conversion shall no longer be required once updated to the latest libp2p. -// Which can has the direct access to the Distance private field of U256. -pub fn convert_distance_to_u256(distance: &Distance) -> U256 { - let addr_str = format!("{distance:?}"); - let numeric_part = addr_str - .trim_start_matches("Distance(") - .trim_end_matches(")") - .to_string(); - let distance_value = U256::from_str(&numeric_part); - distance_value.unwrap_or(U256::ZERO) -} - /// This is the address in the network by which proximity/distance /// to other items (whether nodes or data chunks) are calculated. /// @@ -94,14 +80,14 @@ pub enum NetworkAddress { PeerId(Bytes), /// The NetworkAddress is representing a ChunkAddress. ChunkAddress(ChunkAddress), - /// The NetworkAddress is representing a TransactionAddress. - TransactionAddress(TransactionAddress), - /// The NetworkAddress is representing a ChunkAddress. - RegisterAddress(RegisterAddress), - /// The NetworkAddress is representing a RecordKey. - RecordKey(Bytes), + /// The NetworkAddress is representing a GraphEntryAddress. + GraphEntryAddress(GraphEntryAddress), /// The NetworkAddress is representing a ScratchpadAddress. ScratchpadAddress(ScratchpadAddress), + /// The NetworkAddress is representing a PointerAddress. + PointerAddress(PointerAddress), + /// The NetworkAddress is representing a RecordKey. + RecordKey(Bytes), } impl NetworkAddress { @@ -110,20 +96,16 @@ impl NetworkAddress { NetworkAddress::ChunkAddress(chunk_address) } - /// Return a `NetworkAddress` representation of the `TransactionAddress`. - pub fn from_transaction_address(transaction_address: TransactionAddress) -> Self { - NetworkAddress::TransactionAddress(transaction_address) + /// Return a `NetworkAddress` representation of the `GraphEntryAddress`. + pub fn from_graph_entry_address(graph_entry_address: GraphEntryAddress) -> Self { + NetworkAddress::GraphEntryAddress(graph_entry_address) } - /// Return a `NetworkAddress` representation of the `TransactionAddress`. + + /// Return a `NetworkAddress` representation of the `GraphEntryAddress`. pub fn from_scratchpad_address(address: ScratchpadAddress) -> Self { NetworkAddress::ScratchpadAddress(address) } - /// Return a `NetworkAddress` representation of the `RegisterAddress`. - pub fn from_register_address(register_address: RegisterAddress) -> Self { - NetworkAddress::RegisterAddress(register_address) - } - /// Return a `NetworkAddress` representation of the `PeerId` by encapsulating its bytes. pub fn from_peer(peer_id: PeerId) -> Self { NetworkAddress::PeerId(Bytes::from(peer_id.to_bytes())) @@ -134,18 +116,21 @@ impl NetworkAddress { NetworkAddress::RecordKey(Bytes::copy_from_slice(record_key.as_ref())) } + /// Return a `NetworkAddress` representation of the `PointerAddress`. + pub fn from_pointer_address(pointer_address: PointerAddress) -> Self { + NetworkAddress::PointerAddress(pointer_address) + } + /// Return the encapsulated bytes of this `NetworkAddress`. pub fn as_bytes(&self) -> Vec { match self { NetworkAddress::PeerId(bytes) | NetworkAddress::RecordKey(bytes) => bytes.to_vec(), NetworkAddress::ChunkAddress(chunk_address) => chunk_address.xorname().0.to_vec(), - NetworkAddress::TransactionAddress(transaction_address) => { - transaction_address.xorname().0.to_vec() + NetworkAddress::GraphEntryAddress(graph_entry_address) => { + graph_entry_address.xorname().0.to_vec() } NetworkAddress::ScratchpadAddress(addr) => addr.xorname().0.to_vec(), - NetworkAddress::RegisterAddress(register_address) => { - register_address.xorname().0.to_vec() - } + NetworkAddress::PointerAddress(pointer_address) => pointer_address.0.to_vec(), } } @@ -156,23 +141,9 @@ impl NetworkAddress { return Some(peer_id); } } - None } - /// Try to return the represented `XorName`. - pub fn as_xorname(&self) -> Option { - match self { - NetworkAddress::TransactionAddress(transaction_address) => { - Some(*transaction_address.xorname()) - } - NetworkAddress::ChunkAddress(chunk_address) => Some(*chunk_address.xorname()), - NetworkAddress::RegisterAddress(register_address) => Some(register_address.xorname()), - NetworkAddress::ScratchpadAddress(address) => Some(address.xorname()), - _ => None, - } - } - /// Try to return the represented `RecordKey`. pub fn as_record_key(&self) -> Option { match self { @@ -186,11 +157,11 @@ impl NetworkAddress { match self { NetworkAddress::RecordKey(bytes) => RecordKey::new(bytes), NetworkAddress::ChunkAddress(chunk_address) => RecordKey::new(chunk_address.xorname()), - NetworkAddress::RegisterAddress(register_address) => { - RecordKey::new(®ister_address.xorname()) + NetworkAddress::GraphEntryAddress(graph_entry_address) => { + RecordKey::new(graph_entry_address.xorname()) } - NetworkAddress::TransactionAddress(transaction_address) => { - RecordKey::new(transaction_address.xorname()) + NetworkAddress::PointerAddress(pointer_address) => { + RecordKey::new(pointer_address.xorname()) } NetworkAddress::ScratchpadAddress(addr) => RecordKey::new(&addr.xorname()), NetworkAddress::PeerId(bytes) => RecordKey::new(bytes), @@ -211,16 +182,6 @@ impl NetworkAddress { pub fn distance(&self, other: &NetworkAddress) -> Distance { self.as_kbucket_key().distance(&other.as_kbucket_key()) } - - // NB: Leaving this here as to demonstrate what we can do with this. - // /// Return the uniquely determined key with the given distance to `self`. - // /// - // /// This implements the following equivalence: - // /// - // /// `self xor other = distance <==> other = self xor distance` - // pub fn for_distance(&self, d: Distance) -> libp2p::kad::kbucket::KeyBytes { - // self.as_kbucket_key().for_distance(d) - // } } impl Debug for NetworkAddress { @@ -239,10 +200,10 @@ impl Debug for NetworkAddress { &chunk_address.to_hex()[0..6] ) } - NetworkAddress::TransactionAddress(transaction_address) => { + NetworkAddress::GraphEntryAddress(graph_entry_address) => { format!( - "NetworkAddress::TransactionAddress({} - ", - &transaction_address.to_hex()[0..6] + "NetworkAddress::GraphEntryAddress({} - ", + &graph_entry_address.to_hex()[0..6] ) } NetworkAddress::ScratchpadAddress(scratchpad_address) => { @@ -251,19 +212,21 @@ impl Debug for NetworkAddress { &scratchpad_address.to_hex()[0..6] ) } - NetworkAddress::RegisterAddress(register_address) => format!( - "NetworkAddress::RegisterAddress({} - ", - ®ister_address.to_hex()[0..6] - ), - NetworkAddress::RecordKey(bytes) => format!( - "NetworkAddress::RecordKey({} - ", - &PrettyPrintRecordKey::from(&RecordKey::new(bytes)).no_kbucket_log()[0..6] - ), + NetworkAddress::PointerAddress(pointer_address) => { + format!( + "NetworkAddress::PointerAddress({} - ", + &pointer_address.to_hex()[0..6] + ) + } + NetworkAddress::RecordKey(bytes) => { + format!("NetworkAddress::RecordKey({:?} - ", hex::encode(bytes)) + } }; + write!( f, "{name_str}{:?})", - PrettyPrintKBucketKey(self.as_kbucket_key()), + PrettyPrintKBucketKey(self.as_kbucket_key()) ) } } @@ -277,18 +240,18 @@ impl Display for NetworkAddress { NetworkAddress::ChunkAddress(addr) => { write!(f, "NetworkAddress::ChunkAddress({addr:?})") } - NetworkAddress::TransactionAddress(addr) => { - write!(f, "NetworkAddress::TransactionAddress({addr:?})") + NetworkAddress::GraphEntryAddress(addr) => { + write!(f, "NetworkAddress::GraphEntryAddress({addr:?})") } NetworkAddress::ScratchpadAddress(addr) => { write!(f, "NetworkAddress::ScratchpadAddress({addr:?})") } - NetworkAddress::RegisterAddress(addr) => { - write!(f, "NetworkAddress::RegisterAddress({addr:?})") - } NetworkAddress::RecordKey(key) => { write!(f, "NetworkAddress::RecordKey({})", hex::encode(key)) } + NetworkAddress::PointerAddress(addr) => { + write!(f, "NetworkAddress::PointerAddress({addr:?})") + } } } } @@ -413,19 +376,19 @@ impl std::fmt::Debug for PrettyPrintRecordKey<'_> { #[cfg(test)] mod tests { - use crate::storage::TransactionAddress; + use crate::storage::GraphEntryAddress; use crate::NetworkAddress; use bls::rand::thread_rng; #[test] - fn verify_transaction_addr_is_actionable() { + fn verify_graph_entry_addr_is_actionable() { let xorname = xor_name::XorName::random(&mut thread_rng()); - let transaction_addr = TransactionAddress::new(xorname); - let net_addr = NetworkAddress::from_transaction_address(transaction_addr); + let graph_entry_addr = GraphEntryAddress::new(xorname); + let net_addr = NetworkAddress::from_graph_entry_address(graph_entry_addr); - let transaction_addr_hex = &transaction_addr.to_hex()[0..6]; // we only log the first 6 chars + let graph_entry_addr_hex = &graph_entry_addr.to_hex()[0..6]; // we only log the first 6 chars let net_addr_fmt = format!("{net_addr}"); - assert!(net_addr_fmt.contains(transaction_addr_hex)); + assert!(net_addr_fmt.contains(graph_entry_addr_hex)); } } diff --git a/ant-protocol/src/messages.rs b/ant-protocol/src/messages.rs index cbef76ab90..d79d543daf 100644 --- a/ant-protocol/src/messages.rs +++ b/ant-protocol/src/messages.rs @@ -11,7 +11,6 @@ mod chunk_proof; mod cmd; mod node_id; mod query; -mod register; mod response; pub use self::{ @@ -19,7 +18,6 @@ pub use self::{ cmd::Cmd, node_id::NodeId, query::Query, - register::RegisterCmd, response::{CmdResponse, QueryResponse}, }; diff --git a/ant-protocol/src/messages/cmd.rs b/ant-protocol/src/messages/cmd.rs index f0f5e089b4..c295f73f5b 100644 --- a/ant-protocol/src/messages/cmd.rs +++ b/ant-protocol/src/messages/cmd.rs @@ -7,7 +7,9 @@ // permissions and limitations relating to use of the SAFE Network Software. #![allow(clippy::mutable_key_type)] // for Bytes in NetworkAddress -use crate::{storage::RecordType, NetworkAddress}; +use crate::storage::DataTypes; +use crate::{storage::ValidationType, NetworkAddress}; +use ant_evm::ProofOfPayment; use serde::{Deserialize, Serialize}; /// Ant protocol cmds @@ -25,7 +27,21 @@ pub enum Cmd { /// Holder of the replication keys. holder: NetworkAddress, /// Keys of copy that shall be replicated. - keys: Vec<(NetworkAddress, RecordType)>, + keys: Vec<(NetworkAddress, ValidationType)>, + }, + /// Write operation to notify peer fetch a list of fresh [`NetworkAddress`] from the holder. + /// + /// [`NetworkAddress`]: crate::NetworkAddress + FreshReplicate { + /// Holder of the replication keys. + holder: NetworkAddress, + /// Keys of copy that shall be replicated. + keys: Vec<( + NetworkAddress, + DataTypes, + ValidationType, + Option, + )>, }, /// Notify the peer it is now being considered as BAD due to the included behaviour PeerConsideredAsBad { @@ -46,6 +62,14 @@ impl std::fmt::Debug for Cmd { .field("first_ten_keys", &first_ten_keys) .finish() } + Cmd::FreshReplicate { holder, keys } => { + let first_ten_keys: Vec<_> = keys.iter().take(10).collect(); + f.debug_struct("Cmd::FreshReplicate") + .field("holder", holder) + .field("keys_len", &keys.len()) + .field("first_ten_keys", &first_ten_keys) + .finish() + } Cmd::PeerConsideredAsBad { detected_by, bad_peer, @@ -65,6 +89,7 @@ impl Cmd { pub fn dst(&self) -> NetworkAddress { match self { Cmd::Replicate { holder, .. } => holder.clone(), + Cmd::FreshReplicate { holder, .. } => holder.clone(), Cmd::PeerConsideredAsBad { bad_peer, .. } => bad_peer.clone(), } } @@ -81,6 +106,14 @@ impl std::fmt::Display for Cmd { keys.len() ) } + Cmd::FreshReplicate { holder, keys } => { + write!( + f, + "Cmd::Replicate({:?} has {} keys)", + holder.as_peer_id(), + keys.len() + ) + } Cmd::PeerConsideredAsBad { detected_by, bad_peer, diff --git a/ant-protocol/src/messages/query.rs b/ant-protocol/src/messages/query.rs index f38500bd41..5c2d8a6ac9 100644 --- a/ant-protocol/src/messages/query.rs +++ b/ant-protocol/src/messages/query.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{messages::Nonce, NetworkAddress}; -use ant_evm::U256; +use libp2p::kad::U256; use serde::{Deserialize, Serialize}; /// Data queries - retrieving data and inspecting their structure. @@ -23,6 +23,10 @@ pub enum Query { GetStoreQuote { /// The Address of the record to be stored. key: NetworkAddress, + /// DataTypes as represented as its `index` + data_type: u32, + /// Data size of the record + data_size: usize, /// The random nonce that nodes use to produce the Proof (i.e., hash(record+nonce)) /// Set to None if no need to carry out storage check. nonce: Option, @@ -42,17 +46,6 @@ pub enum Query { /// Key of the record to be fetched key: NetworkAddress, }, - /// Retrieve a specific register record from a specific peer. - /// - /// This should eventually lead to a [`GetRegisterRecord`] response. - /// - /// [`GetRegisterRecord`]: super::QueryResponse::GetRegisterRecord - GetRegisterRecord { - /// Sender of the query - requester: NetworkAddress, - /// Key of the register record to be fetched - key: NetworkAddress, - }, /// Get the proof that the chunk with the given NetworkAddress exists with the requested node. GetChunkExistenceProof { /// The Address of the chunk that we are trying to verify. @@ -66,7 +59,7 @@ pub enum Query { }, /// Queries close_group peers whether the target peer is a bad_node CheckNodeInProblem(NetworkAddress), - /// Query the the peers in range to the target address, from the receiver's perspective. + /// Query the peers in range to the target address, from the receiver's perspective. /// In case none of the parameters provided, returns nothing. /// In case both of the parameters provided, `range` is preferred to be replied. GetClosestPeers { @@ -89,7 +82,6 @@ impl Query { // and the destination shall be decided by the requester already. Query::GetStoreQuote { key, .. } | Query::GetReplicatedRecord { key, .. } - | Query::GetRegisterRecord { key, .. } | Query::GetChunkExistenceProof { key, .. } | Query::GetClosestPeers { key, .. } => key.clone(), } @@ -101,17 +93,19 @@ impl std::fmt::Display for Query { match self { Query::GetStoreQuote { key, + data_type, + data_size, nonce, difficulty, } => { - write!(f, "Query::GetStoreQuote({key:?} {nonce:?} {difficulty})") + write!( + f, + "Query::GetStoreQuote({key:?} {data_type} {data_size} {nonce:?} {difficulty})" + ) } Query::GetReplicatedRecord { key, requester } => { write!(f, "Query::GetReplicatedRecord({requester:?} {key:?})") } - Query::GetRegisterRecord { key, requester } => { - write!(f, "Query::GetRegisterRecord({requester:?} {key:?})") - } Query::GetChunkExistenceProof { key, nonce, @@ -131,7 +125,7 @@ impl std::fmt::Display for Query { range, sign_result, } => { - let distance = range.as_ref().map(|value| U256::from_be_slice(value)); + let distance = range.as_ref().map(|value| U256::from_big_endian(value)); write!( f, "Query::GetClosestPeers({key:?} {num_of_peers:?} {distance:?} {sign_result})" diff --git a/ant-protocol/src/messages/register.rs b/ant-protocol/src/messages/register.rs deleted file mode 100644 index bd57791aaf..0000000000 --- a/ant-protocol/src/messages/register.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use ant_registers::{Register, RegisterAddress, RegisterOp}; - -use serde::{Deserialize, Serialize}; - -/// A register cmd that is sent over to the Network -#[derive(Eq, PartialEq, Clone, Serialize, Deserialize)] -pub enum RegisterCmd { - /// Create a new register on the network. - Create { - /// The base register (contains, owner, name, tag, permissions, and register initial state) - register: Register, - /// The signature of the owner on that register. - signature: bls::Signature, - }, - /// Edit the register - Edit(RegisterOp), -} - -/// Custom debug implementation to avoid printing the whole register -/// instead we just print the address -impl std::fmt::Debug for RegisterCmd { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - RegisterCmd::Create { register, .. } => { - write!(f, "RegisterCmd::Create({:?})", register.address()) - } - RegisterCmd::Edit(op) => write!(f, "RegisterCmd::Edit({:?})", op.address()), - } - } -} - -impl RegisterCmd { - /// Returns the dst address of the register. - pub fn dst(&self) -> RegisterAddress { - match self { - Self::Create { register, .. } => *register.address(), - Self::Edit(op) => op.address(), - } - } -} diff --git a/ant-protocol/src/messages/response.rs b/ant-protocol/src/messages/response.rs index 48b332c60b..bd2d6364cd 100644 --- a/ant-protocol/src/messages/response.rs +++ b/ant-protocol/src/messages/response.rs @@ -46,12 +46,6 @@ pub enum QueryResponse { /// /// [`GetReplicatedRecord`]: crate::messages::Query::GetReplicatedRecord GetReplicatedRecord(Result<(NetworkAddress, Bytes)>), - // ===== RegisterRecord ===== - // - /// Response to [`GetRegisterRecord`] - /// - /// [`GetRegisterRecord`]: crate::messages::Query::GetRegisterRecord - GetRegisterRecord(Result<(NetworkAddress, Bytes)>), // ===== ChunkExistenceProof ===== // /// Response to [`GetChunkExistenceProof`] @@ -113,19 +107,6 @@ impl Debug for QueryResponse { write!(f, "GetReplicatedRecord(Err({err:?}))") } }, - QueryResponse::GetRegisterRecord(result) => match result { - Ok((holder, data)) => { - write!( - f, - "GetRegisterRecord(Ok((holder: {:?}, datalen: {:?})))", - holder, - data.len() - ) - } - Err(err) => { - write!(f, "GetRegisterRecord(Err({err:?}))") - } - }, QueryResponse::GetChunkExistenceProof(proofs) => { let addresses: Vec<_> = proofs.iter().map(|(addr, _)| addr.clone()).collect(); write!(f, "GetChunkExistenceProof(checked chunks: {addresses:?})") @@ -149,6 +130,8 @@ pub enum CmdResponse { // /// Response to replication cmd Replicate(Result<()>), + /// Response to fresh replication cmd + FreshReplicate(Result<()>), // // ===== PeerConsideredAsBad ===== // diff --git a/ant-protocol/src/storage.rs b/ant-protocol/src/storage.rs deleted file mode 100644 index 9d3e675039..0000000000 --- a/ant-protocol/src/storage.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -mod address; -mod chunks; -mod header; -mod scratchpad; -mod transaction; - -use core::fmt; -use exponential_backoff::Backoff; -use std::{num::NonZeroUsize, time::Duration}; - -pub use self::{ - address::{ChunkAddress, RegisterAddress, ScratchpadAddress, TransactionAddress}, - chunks::Chunk, - header::{try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, RecordType}, - scratchpad::Scratchpad, - transaction::Transaction, -}; - -/// A strategy that translates into a configuration for exponential backoff. -/// The first retry is done after 2 seconds, after which the backoff is roughly doubled each time. -/// The interval does not go beyond 32 seconds. So the intervals increase from 2 to 4, to 8, to 16, to 32 seconds and -/// all attempts are made at most 32 seconds apart. -/// -/// The exact timings depend on jitter, which is set to 0.2, meaning the intervals can deviate quite a bit -/// from the ones listed in the docs. -#[derive(Clone, Debug, Copy, Default)] -pub enum RetryStrategy { - /// Attempt once (no retries) - None, - /// Retry 3 times (waits 2s, 4s and lastly 8s; max total time ~14s) - Quick, - /// Retry 5 times (waits 2s, 4s, 8s, 16s and lastly 32s; max total time ~62s) - #[default] - Balanced, - /// Retry 9 times (waits 2s, 4s, 8s, 16s, 32s, 32s, 32s, 32s and lastly 32s; max total time ~190s) - Persistent, - /// Attempt a specific number of times - N(NonZeroUsize), -} - -impl RetryStrategy { - pub fn attempts(&self) -> usize { - match self { - RetryStrategy::None => 1, - RetryStrategy::Quick => 4, - RetryStrategy::Balanced => 6, - RetryStrategy::Persistent => 10, - RetryStrategy::N(x) => x.get(), - } - } - - pub fn backoff(&self) -> Backoff { - let mut backoff = Backoff::new( - self.attempts() as u32, - Duration::from_secs(1), // First interval is double of this (see https://github.com/yoshuawuyts/exponential-backoff/issues/23) - Some(Duration::from_secs(32)), - ); - backoff.set_factor(2); // Default. - backoff.set_jitter(0.2); // Default is 0.3. - backoff - } -} - -impl fmt::Display for RetryStrategy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self:?}") - } -} - -#[test] -fn verify_retry_strategy_intervals() { - let intervals = |strategy: RetryStrategy| -> Vec { - let mut backoff = strategy.backoff(); - backoff.set_jitter(0.01); // Make intervals deterministic. - backoff - .into_iter() - .flatten() - .map(|duration| duration.as_secs_f64().round() as u32) - .collect() - }; - - assert_eq!(intervals(RetryStrategy::None), Vec::::new()); - assert_eq!(intervals(RetryStrategy::Quick), vec![2, 4, 8]); - assert_eq!(intervals(RetryStrategy::Balanced), vec![2, 4, 8, 16, 32]); - assert_eq!( - intervals(RetryStrategy::Persistent), - vec![2, 4, 8, 16, 32, 32, 32, 32, 32] - ); - assert_eq!( - intervals(RetryStrategy::N(NonZeroUsize::new(12).unwrap())), - vec![2, 4, 8, 16, 32, 32, 32, 32, 32, 32, 32] - ); -} diff --git a/ant-protocol/src/storage/address/transaction.rs b/ant-protocol/src/storage/address/graph.rs similarity index 81% rename from ant-protocol/src/storage/address/transaction.rs rename to ant-protocol/src/storage/address/graph.rs index 399a7a6397..4a247f76f6 100644 --- a/ant-protocol/src/storage/address/transaction.rs +++ b/ant-protocol/src/storage/address/graph.rs @@ -10,11 +10,11 @@ use bls::PublicKey; use serde::{Deserialize, Serialize}; use xor_name::XorName; -/// Address of a transaction, is derived from the owner's public key +/// Address of a GraphEntry, is derived from the owner's unique public key #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub struct TransactionAddress(pub XorName); +pub struct GraphEntryAddress(pub XorName); -impl TransactionAddress { +impl GraphEntryAddress { pub fn from_owner(owner: PublicKey) -> Self { Self(XorName::from_content(&owner.to_bytes())) } @@ -32,8 +32,8 @@ impl TransactionAddress { } } -impl std::fmt::Debug for TransactionAddress { +impl std::fmt::Debug for GraphEntryAddress { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "TransactionAddress({})", &self.to_hex()[0..6]) + write!(f, "GraphEntryAddress({})", &self.to_hex()[0..6]) } } diff --git a/ant-protocol/src/storage/address/mod.rs b/ant-protocol/src/storage/address/mod.rs new file mode 100644 index 0000000000..f1bf8abd4a --- /dev/null +++ b/ant-protocol/src/storage/address/mod.rs @@ -0,0 +1,9 @@ +pub mod chunk; +pub mod graph; +pub mod pointer_address; +pub mod scratchpad; + +pub use chunk::ChunkAddress; +pub use graph::GraphEntryAddress; +pub use pointer_address::PointerAddress; +pub use scratchpad::ScratchpadAddress; diff --git a/ant-protocol/src/storage/address/pointer_address.rs b/ant-protocol/src/storage/address/pointer_address.rs new file mode 100644 index 0000000000..c6406f4889 --- /dev/null +++ b/ant-protocol/src/storage/address/pointer_address.rs @@ -0,0 +1,53 @@ +use bls::PublicKey; +use serde::{Deserialize, Serialize}; +use xor_name::XorName; + +/// Address of a pointer, is derived from the owner's public key +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct PointerAddress(pub XorName); + +impl PointerAddress { + pub fn from_owner(owner: PublicKey) -> Self { + Self(XorName::from_content(&owner.to_bytes())) + } + + pub fn new(xor_name: XorName) -> Self { + Self(xor_name) + } + + pub fn xorname(&self) -> &XorName { + &self.0 + } + + pub fn to_hex(&self) -> String { + hex::encode(self.0) + } + + pub fn to_bytes(&self) -> Vec { + rmp_serde::to_vec(self).expect("Failed to serialize PointerAddress") + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + rmp_serde::from_slice(bytes) + } +} + +impl std::fmt::Debug for PointerAddress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PointerAddress({})", &self.to_hex()[0..6]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pointer_serialization() { + let key = bls::SecretKey::random(); + let pointer_address = PointerAddress::from_owner(key.public_key()); + let serialized = pointer_address.to_bytes(); + let deserialized = PointerAddress::from_bytes(&serialized).unwrap(); + assert_eq!(pointer_address, deserialized); + } +} diff --git a/ant-protocol/src/storage/chunks.rs b/ant-protocol/src/storage/chunks.rs index 7457e5a845..a97ce92937 100644 --- a/ant-protocol/src/storage/chunks.rs +++ b/ant-protocol/src/storage/chunks.rs @@ -25,6 +25,9 @@ pub struct Chunk { } impl Chunk { + /// The default maximum size of a chunk is 1MB + pub const DEFAULT_MAX_SIZE: usize = 1024 * 1024; + /// Creates a new instance of `Chunk`. pub fn new(value: Bytes) -> Self { Self { @@ -53,14 +56,14 @@ impl Chunk { self.address.xorname() } - /// Returns size of contained value. - pub fn payload_size(&self) -> usize { + /// Returns size of this chunk after serialisation. + pub fn size(&self) -> usize { self.value.len() } - /// Returns size of this chunk after serialisation. - pub fn serialised_size(&self) -> usize { - self.value.len() + /// Returns true if the chunk is too big + pub fn is_too_big(&self) -> bool { + self.size() > Self::DEFAULT_MAX_SIZE } } diff --git a/ant-protocol/src/storage/graph.rs b/ant-protocol/src/storage/graph.rs new file mode 100644 index 0000000000..c647131bd1 --- /dev/null +++ b/ant-protocol/src/storage/graph.rs @@ -0,0 +1,149 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use super::address::GraphEntryAddress; +use bls::SecretKey; +use serde::{Deserialize, Serialize}; + +// re-exports +pub use bls::{PublicKey, Signature}; + +/// Content of a graph, limited to 32 bytes +pub type GraphContent = [u8; 32]; + +/// A generic GraphEntry on the Network. +/// +/// Graph entries are stored at the owner's public key. Note that there can only be one graph entry per owner. +/// Graph entries can be linked to other graph entries as parents or descendants. +/// Applications are free to define the meaning of these links, those are not enforced by the protocol. +/// The protocol only ensures that the graph entry is immutable once uploaded and that the signature is valid and matches the owner. +/// +/// For convenience it is advised to make use of BLS key derivation to create multiple graph entries from a single key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Ord, PartialOrd)] +pub struct GraphEntry { + /// The owner of the graph. Note that graph entries are stored at the owner's public key + pub owner: PublicKey, + /// Other graph entries that this graph entry refers to as parents + pub parents: Vec, + /// The content of the graph entry + pub content: GraphContent, + /// Other graph entries that this graph entry refers to as descendants/outputs along with some data associated to each one + pub descendants: Vec<(PublicKey, GraphContent)>, + /// signs the above 4 fields with the owners key + pub signature: Signature, +} + +impl GraphEntry { + /// Maximum size of a graph entry: 100KB + pub const MAX_SIZE: usize = 100 * 1024; + + /// Create a new graph entry, signing it with the provided secret key. + pub fn new( + owner: &SecretKey, + parents: Vec, + content: GraphContent, + descendants: Vec<(PublicKey, GraphContent)>, + ) -> Self { + let key = owner; + let owner = key.public_key(); + let signature = key.sign(Self::bytes_to_sign( + &owner, + &parents, + &content, + &descendants, + )); + Self { + owner, + parents, + content, + descendants, + signature, + } + } + + /// Create a new graph entry, with the signature already calculated. + pub fn new_with_signature( + owner: PublicKey, + parents: Vec, + content: GraphContent, + descendants: Vec<(PublicKey, GraphContent)>, + signature: Signature, + ) -> Self { + Self { + owner, + parents, + content, + descendants, + signature, + } + } + + /// Get the bytes that the signature is calculated from. + pub fn bytes_to_sign( + owner: &PublicKey, + parents: &[PublicKey], + content: &[u8], + descendants: &[(PublicKey, GraphContent)], + ) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&owner.to_bytes()); + bytes.extend_from_slice("parent".as_bytes()); + bytes.extend_from_slice( + &parents + .iter() + .map(|p| p.to_bytes()) + .collect::>() + .concat(), + ); + bytes.extend_from_slice("content".as_bytes()); + bytes.extend_from_slice(content); + bytes.extend_from_slice("descendants".as_bytes()); + bytes.extend_from_slice( + &descendants + .iter() + .flat_map(|(p, c)| [&p.to_bytes(), c.as_slice()].concat()) + .collect::>(), + ); + bytes + } + + pub fn address(&self) -> GraphEntryAddress { + GraphEntryAddress::from_owner(self.owner) + } + + /// Get the bytes that the signature is calculated from. + pub fn bytes_for_signature(&self) -> Vec { + Self::bytes_to_sign(&self.owner, &self.parents, &self.content, &self.descendants) + } + + /// Verify the signature of the graph entry + pub fn verify_signature(&self) -> bool { + self.owner + .verify(&self.signature, self.bytes_for_signature()) + } + + /// Size of the graph entry + pub fn size(&self) -> usize { + size_of::() + + self + .descendants + .iter() + .map(|(p, c)| p.to_bytes().len() + c.len()) + .sum::() + + self + .parents + .iter() + .map(|p| p.to_bytes().len()) + .sum::() + } + + /// Returns true if the graph entry is too big + pub fn is_too_big(&self) -> bool { + self.size() > Self::MAX_SIZE + } +} diff --git a/ant-protocol/src/storage/header.rs b/ant-protocol/src/storage/header.rs index 7cfd2ffedf..d932f1f19a 100644 --- a/ant-protocol/src/storage/header.rs +++ b/ant-protocol/src/storage/header.rs @@ -10,18 +10,48 @@ use crate::error::Error; use crate::PrettyPrintRecordKey; use bytes::{BufMut, Bytes, BytesMut}; use libp2p::kad::Record; +use prometheus_client::encoding::EncodeLabelValue; use rmp_serde::Serializer; use serde::{Deserialize, Serialize}; use std::fmt::Display; use xor_name::XorName; +/// Data types that natively suppported by autonomi network. +#[derive(EncodeLabelValue, Debug, Serialize, Deserialize, Clone, Copy, Eq, PartialEq, Hash)] +pub enum DataTypes { + Chunk, + GraphEntry, + Pointer, + Scratchpad, +} + +impl DataTypes { + pub fn get_index(&self) -> u32 { + match self { + Self::Chunk => 0, + Self::GraphEntry => 1, + Self::Pointer => 2, + Self::Scratchpad => 3, + } + } + + pub fn from_index(index: u32) -> Option { + match index { + 0 => Some(Self::Chunk), + 1 => Some(Self::GraphEntry), + 2 => Some(Self::Pointer), + 3 => Some(Self::Scratchpad), + _ => None, + } + } +} + /// Indicates the type of the record content. -/// Note for `Spend` and `Register`, using its content_hash (in `XorName` format) -/// to indicate different content body. +/// This is to be only used within the node instance to reflect different content version. +/// Hence, only need to have two entries: Chunk and NonChunk. #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] -pub enum RecordType { +pub enum ValidationType { Chunk, - Scratchpad, NonChunk(XorName), } @@ -30,33 +60,28 @@ pub struct RecordHeader { pub kind: RecordKind, } +/// To be used between client and nodes, hence need to indicate whehter payment info involved. #[derive(Debug, Eq, PartialEq, Clone, Copy)] pub enum RecordKind { - Chunk, - ChunkWithPayment, - Transaction, - TransactionWithPayment, - Register, - RegisterWithPayment, - Scratchpad, - ScratchpadWithPayment, + DataOnly(DataTypes), + DataWithPayment(DataTypes), } +/// Allowing 10 data types to be defined, leaving margin for future. +pub const RECORD_KIND_PAYMENT_STARTING_INDEX: u32 = 10; + impl Serialize for RecordKind { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { - match *self { - Self::ChunkWithPayment => serializer.serialize_u32(0), - Self::Chunk => serializer.serialize_u32(1), - Self::Transaction => serializer.serialize_u32(2), - Self::Register => serializer.serialize_u32(3), - Self::RegisterWithPayment => serializer.serialize_u32(4), - Self::Scratchpad => serializer.serialize_u32(5), - Self::ScratchpadWithPayment => serializer.serialize_u32(6), - Self::TransactionWithPayment => serializer.serialize_u32(7), - } + let index = match self { + Self::DataOnly(ref data_types) => data_types.get_index(), + Self::DataWithPayment(ref data_types) => { + RECORD_KIND_PAYMENT_STARTING_INDEX + data_types.get_index() + } + }; + serializer.serialize_u32(index) } } @@ -66,18 +91,22 @@ impl<'de> Deserialize<'de> for RecordKind { D: serde::Deserializer<'de>, { let num = u32::deserialize(deserializer)?; - match num { - 0 => Ok(Self::ChunkWithPayment), - 1 => Ok(Self::Chunk), - 2 => Ok(Self::Transaction), - 3 => Ok(Self::Register), - 4 => Ok(Self::RegisterWithPayment), - 5 => Ok(Self::Scratchpad), - 6 => Ok(Self::ScratchpadWithPayment), - 7 => Ok(Self::TransactionWithPayment), - _ => Err(serde::de::Error::custom( - "Unexpected integer for RecordKind variant", - )), + let data_type_index = if num < RECORD_KIND_PAYMENT_STARTING_INDEX { + num + } else { + num - RECORD_KIND_PAYMENT_STARTING_INDEX + }; + + if let Some(data_type) = DataTypes::from_index(data_type_index) { + if num < RECORD_KIND_PAYMENT_STARTING_INDEX { + Ok(Self::DataOnly(data_type)) + } else { + Ok(Self::DataWithPayment(data_type)) + } + } else { + Err(serde::de::Error::custom(format!( + "Unexpected index {num} for RecordKind variant", + ))) } } } @@ -121,7 +150,16 @@ impl RecordHeader { pub fn is_record_of_type_chunk(record: &Record) -> Result { let kind = Self::from_record(record)?.kind; - Ok(kind == RecordKind::Chunk) + Ok(kind == RecordKind::DataOnly(DataTypes::Chunk)) + } + + pub fn get_data_type(record: &Record) -> Result { + let kind = Self::from_record(record)?.kind; + match kind { + RecordKind::DataOnly(data_type) | RecordKind::DataWithPayment(data_type) => { + Ok(data_type) + } + } } } @@ -160,53 +198,78 @@ pub fn try_serialize_record( #[cfg(test)] mod tests { - use super::{RecordHeader, RecordKind}; + use super::*; use crate::error::Result; #[test] fn verify_record_header_encoded_size() -> Result<()> { let chunk_with_payment = RecordHeader { - kind: RecordKind::ChunkWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Chunk), } .try_serialize()?; assert_eq!(chunk_with_payment.len(), RecordHeader::SIZE); - let reg_with_payment = RecordHeader { - kind: RecordKind::RegisterWithPayment, - } - .try_serialize()?; - assert_eq!(reg_with_payment.len(), RecordHeader::SIZE); - let chunk = RecordHeader { - kind: RecordKind::Chunk, + kind: RecordKind::DataOnly(DataTypes::Chunk), } .try_serialize()?; assert_eq!(chunk.len(), RecordHeader::SIZE); - let transaction = RecordHeader { - kind: RecordKind::Transaction, - } - .try_serialize()?; - assert_eq!(transaction.len(), RecordHeader::SIZE); - - let register = RecordHeader { - kind: RecordKind::Register, + let graphentry = RecordHeader { + kind: RecordKind::DataOnly(DataTypes::GraphEntry), } .try_serialize()?; - assert_eq!(register.len(), RecordHeader::SIZE); + assert_eq!(graphentry.len(), RecordHeader::SIZE); let scratchpad = RecordHeader { - kind: RecordKind::Scratchpad, + kind: RecordKind::DataOnly(DataTypes::Scratchpad), } .try_serialize()?; assert_eq!(scratchpad.len(), RecordHeader::SIZE); let scratchpad_with_payment = RecordHeader { - kind: RecordKind::ScratchpadWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Scratchpad), } .try_serialize()?; assert_eq!(scratchpad_with_payment.len(), RecordHeader::SIZE); + let pointer = RecordHeader { + kind: RecordKind::DataOnly(DataTypes::Pointer), + } + .try_serialize()?; + assert_eq!(pointer.len(), RecordHeader::SIZE); + + let pointer_with_payment = RecordHeader { + kind: RecordKind::DataWithPayment(DataTypes::Pointer), + } + .try_serialize()?; + assert_eq!(pointer_with_payment.len(), RecordHeader::SIZE); + + Ok(()) + } + + #[test] + fn test_record_kind_serialization() -> Result<()> { + let kinds = vec![ + RecordKind::DataOnly(DataTypes::Chunk), + RecordKind::DataWithPayment(DataTypes::Chunk), + RecordKind::DataOnly(DataTypes::GraphEntry), + RecordKind::DataWithPayment(DataTypes::GraphEntry), + RecordKind::DataOnly(DataTypes::Scratchpad), + RecordKind::DataWithPayment(DataTypes::Scratchpad), + RecordKind::DataOnly(DataTypes::Pointer), + RecordKind::DataWithPayment(DataTypes::Pointer), + ]; + + for kind in kinds { + let header = RecordHeader { kind }; + let header2 = RecordHeader { kind }; + + let serialized = header.try_serialize()?; + let deserialized = RecordHeader::try_deserialize(&serialized)?; + assert_eq!(header2.kind, deserialized.kind); + } + Ok(()) } } diff --git a/ant-registers/src/lib.rs b/ant-protocol/src/storage/mod.rs similarity index 57% rename from ant-registers/src/lib.rs rename to ant-protocol/src/storage/mod.rs index e9cc34e4f0..6fc76b34e8 100644 --- a/ant-registers/src/lib.rs +++ b/ant-protocol/src/storage/mod.rs @@ -7,19 +7,20 @@ // permissions and limitations relating to use of the SAFE Network Software. mod address; -pub(crate) mod error; -mod metadata; -mod permissions; -pub(crate) mod reg_crdt; -pub(crate) mod register; -mod register_op; +mod chunks; +mod graph; +mod header; +mod pointer; +mod scratchpad; pub use self::{ - address::RegisterAddress, - error::Error, - metadata::{Entry, EntryHash}, - permissions::Permissions, - reg_crdt::RegisterCrdt, - register::{Register, SignedRegister}, - register_op::RegisterOp, + address::{ChunkAddress, GraphEntryAddress, PointerAddress, ScratchpadAddress}, + chunks::Chunk, + graph::{GraphContent, GraphEntry}, + header::{ + try_deserialize_record, try_serialize_record, DataTypes, RecordHeader, RecordKind, + ValidationType, + }, + pointer::{Pointer, PointerTarget}, + scratchpad::Scratchpad, }; diff --git a/ant-protocol/src/storage/pointer.rs b/ant-protocol/src/storage/pointer.rs new file mode 100644 index 0000000000..ddfdb5ade1 --- /dev/null +++ b/ant-protocol/src/storage/pointer.rs @@ -0,0 +1,155 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::storage::{ChunkAddress, GraphEntryAddress, PointerAddress, ScratchpadAddress}; +use bls::{PublicKey, SecretKey, Signature}; +use serde::{Deserialize, Serialize}; +use xor_name::XorName; + +/// Pointer, a mutable address pointing to other data on the Network +/// It is stored at the owner's public key and can only be updated by the owner +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Ord, PartialOrd)] +pub struct Pointer { + owner: PublicKey, + counter: u32, + target: PointerTarget, + signature: Signature, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Ord, PartialOrd)] +pub enum PointerTarget { + ChunkAddress(ChunkAddress), + GraphEntryAddress(GraphEntryAddress), + PointerAddress(PointerAddress), + ScratchpadAddress(ScratchpadAddress), +} + +impl PointerTarget { + pub fn xorname(&self) -> XorName { + match self { + PointerTarget::ChunkAddress(addr) => *addr.xorname(), + PointerTarget::GraphEntryAddress(addr) => *addr.xorname(), + PointerTarget::PointerAddress(ptr) => *ptr.xorname(), + PointerTarget::ScratchpadAddress(addr) => addr.xorname(), + } + } +} + +impl Pointer { + /// Create a new pointer, signing it with the provided secret key. + /// This pointer would be stored on the network at the provided key's public key. + /// There can only be one pointer at a time at the same address (one per key). + pub fn new(owner: &SecretKey, counter: u32, target: PointerTarget) -> Self { + let pubkey = owner.public_key(); + let bytes_to_sign = Self::bytes_to_sign(&pubkey, counter, &target); + let signature = owner.sign(&bytes_to_sign); + + Self { + owner: pubkey, + counter, + target, + signature, + } + } + + /// Create a new pointer with an existing signature + pub fn new_with_signature( + owner: PublicKey, + counter: u32, + target: PointerTarget, + signature: Signature, + ) -> Self { + Self { + owner, + counter, + target, + signature, + } + } + + /// Get the bytes that the signature is calculated from + fn bytes_to_sign(owner: &PublicKey, counter: u32, target: &PointerTarget) -> Vec { + let mut bytes = Vec::new(); + // Add owner public key bytes + bytes.extend_from_slice(&owner.to_bytes()); + // Add counter + bytes.extend_from_slice(&counter.to_le_bytes()); + // Add target bytes using MessagePack serialization + if let Ok(target_bytes) = rmp_serde::to_vec(target) { + bytes.extend_from_slice(&target_bytes); + } + bytes + } + + /// Get the address of the pointer + pub fn address(&self) -> PointerAddress { + PointerAddress::from_owner(self.owner) + } + + /// Get the owner of the pointer + pub fn owner(&self) -> &PublicKey { + &self.owner + } + + /// Get the target of the pointer + pub fn target(&self) -> &PointerTarget { + &self.target + } + + /// Get the bytes that were signed for this pointer + pub fn bytes_for_signature(&self) -> Vec { + Self::bytes_to_sign(&self.owner, self.counter, &self.target) + } + + pub fn xorname(&self) -> XorName { + self.target.xorname() + } + + /// Get the counter of the pointer, the higher the counter, the more recent the pointer is + /// Similarly to counter CRDTs only the latest version (highest counter) of the pointer is kept on the network + pub fn counter(&self) -> u32 { + self.counter + } + + /// Verifies if the pointer has a valid signature + pub fn verify_signature(&self) -> bool { + let bytes = self.bytes_for_signature(); + self.owner.verify(&self.signature, &bytes) + } + + /// Size of the pointer + pub fn size() -> usize { + size_of::() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::thread_rng; + + #[test] + fn test_pointer_creation_and_validation() { + let owner_sk = SecretKey::random(); + let counter = 1; + let mut rng = thread_rng(); + let target = + PointerTarget::GraphEntryAddress(GraphEntryAddress::new(XorName::random(&mut rng))); + + // Create and sign pointer + let pointer = Pointer::new(&owner_sk, counter, target.clone()); + assert!(pointer.verify_signature()); // Should be valid with correct signature + + // Create pointer with wrong signature + let wrong_sk = SecretKey::random(); + let sig = wrong_sk.sign(pointer.bytes_for_signature()); + let wrong_pointer = + Pointer::new_with_signature(owner_sk.public_key(), counter, target.clone(), sig); + assert!(!wrong_pointer.verify_signature()); // Should be invalid with wrong signature + } +} diff --git a/ant-protocol/src/storage/scratchpad.rs b/ant-protocol/src/storage/scratchpad.rs index 1022941de2..e1e76f660c 100644 --- a/ant-protocol/src/storage/scratchpad.rs +++ b/ant-protocol/src/storage/scratchpad.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use xor_name::XorName; -/// Scratchpad, an mutable address for encrypted data +/// Scratchpad, a mutable space for encrypted data on the Network #[derive( Hash, Eq, PartialEq, PartialOrd, Ord, Clone, custom_debug::Debug, Serialize, Deserialize, )] @@ -25,30 +25,81 @@ pub struct Scratchpad { address: ScratchpadAddress, /// Data encoding: custom apps using scratchpad should use this so they can identify the type of data they are storing data_encoding: u64, - /// Contained data. This should be encrypted + /// Encrypted data stored in the scratchpad, it is encrypted automatically by the [`Scratchpad::new`] and [`Scratchpad::update`] methods #[debug(skip)] encrypted_data: Bytes, /// Monotonically increasing counter to track the number of times this has been updated. + /// When pushed to the network, the scratchpad with the highest counter is kept. counter: u64, - /// Signature over `Vec`.extend(Xorname::from_content(encrypted_data).to_vec()) from the owning key. - /// Required for scratchpad to be valid. - signature: Option, + /// Signature over the above fields + signature: Signature, } impl Scratchpad { - /// Creates a new instance of `Scratchpad`. - pub fn new(owner: PublicKey, data_encoding: u64) -> Self { + /// Max Scratchpad size is 4MB including the metadata + pub const MAX_SIZE: usize = 4 * 1024 * 1024; + + /// Creates a new instance of `Scratchpad`. Encrypts the data, and signs all the elements. + pub fn new( + owner: &SecretKey, + data_encoding: u64, + unencrypted_data: &Bytes, + counter: u64, + ) -> Self { + let pk = owner.public_key(); + let encrypted_data = Bytes::from(pk.encrypt(unencrypted_data).to_bytes()); + let addr = ScratchpadAddress::new(pk); + let signature = owner.sign(Self::bytes_for_signature( + addr, + data_encoding, + &encrypted_data, + counter, + )); + Self { + address: addr, + encrypted_data, + data_encoding, + counter, + signature, + } + } + + /// Create a new Scratchpad without provding the secret key + /// It is the caller's responsibility to ensure the signature is valid (signs [`Scratchpad::bytes_for_signature`]) and the data is encrypted + /// It is recommended to use the [`Scratchpad::new`] method instead when possible + pub fn new_with_signature( + owner: PublicKey, + data_encoding: u64, + encrypted_data: Bytes, + counter: u64, + signature: Signature, + ) -> Self { Self { address: ScratchpadAddress::new(owner), - encrypted_data: Bytes::new(), + encrypted_data, data_encoding, - counter: 0, - signature: None, + counter, + signature, } } - /// Return the current count - pub fn count(&self) -> u64 { + /// Returns the bytes to sign for the signature + pub fn bytes_for_signature( + address: ScratchpadAddress, + data_encoding: u64, + encrypted_data: &Bytes, + counter: u64, + ) -> Vec { + let mut bytes_to_sign = data_encoding.to_be_bytes().to_vec(); + bytes_to_sign.extend(address.to_hex().as_bytes()); + bytes_to_sign.extend(counter.to_be_bytes().to_vec()); + bytes_to_sign.extend(encrypted_data.to_vec()); + bytes_to_sign + } + + /// Get the counter of the Scratchpad, the higher the counter, the more recent the Scratchpad is + /// Similarly to counter CRDTs only the latest version (highest counter) of the Scratchpad is kept on the network + pub fn counter(&self) -> u64 { self.counter } @@ -57,43 +108,32 @@ impl Scratchpad { self.data_encoding } - /// Increments the counter value. - pub fn increment(&mut self) -> u64 { + /// Updates the content and encrypts it, increments the counter, re-signs the scratchpad + pub fn update(&mut self, unencrypted_data: &Bytes, sk: &SecretKey) { self.counter += 1; - - self.counter - } - - /// Returns the next counter value, - /// - /// Encrypts data and updates the signature with provided sk - pub fn update_and_sign(&mut self, unencrypted_data: Bytes, sk: &SecretKey) -> u64 { - let next_count = self.increment(); - let pk = self.owner(); - + let address = ScratchpadAddress::new(*pk); self.encrypted_data = Bytes::from(pk.encrypt(unencrypted_data).to_bytes()); - let encrypted_data_xorname = self.encrypted_data_hash().to_vec(); - - let mut bytes_to_sign = self.counter.to_be_bytes().to_vec(); - bytes_to_sign.extend(encrypted_data_xorname); - - self.signature = Some(sk.sign(&bytes_to_sign)); - next_count + let bytes_to_sign = Self::bytes_for_signature( + address, + self.data_encoding, + &self.encrypted_data, + self.counter, + ); + self.signature = sk.sign(&bytes_to_sign); + debug_assert!(self.verify_signature(), "Must be valid after being signed. This is a bug, please report it by opening an issue on our github"); } - /// Verifies the signature and content of the scratchpad are valid for the - /// owner's public key. - pub fn is_valid(&self) -> bool { - if let Some(signature) = &self.signature { - let mut signing_bytes = self.counter.to_be_bytes().to_vec(); - signing_bytes.extend(self.encrypted_data_hash().to_vec()); // add the count - - self.owner().verify(signature, &signing_bytes) - } else { - false - } + /// Verifies that the Scratchpad signature is valid + pub fn verify_signature(&self) -> bool { + let signing_bytes = Self::bytes_for_signature( + self.address, + self.data_encoding, + &self.encrypted_data, + self.counter, + ); + self.owner().verify(&self.signature, &signing_bytes) } /// Returns the encrypted_data. @@ -116,12 +156,12 @@ impl Scratchpad { XorName::from_content(&self.encrypted_data) } - /// Returns the owner. + /// Returns the owner of the scratchpad pub fn owner(&self) -> &PublicKey { self.address.owner() } - /// Returns the address. + /// Returns the address of the scratchpad pub fn address(&self) -> &ScratchpadAddress { &self.address } @@ -131,16 +171,8 @@ impl Scratchpad { NetworkAddress::ScratchpadAddress(self.address) } - /// Returns a VEC with the XOR name. - pub fn to_xor_name_vec(&self) -> Vec { - [self.network_address()] - .iter() - .filter_map(|f| f.as_xorname()) - .collect::>() - } - - /// Returns the name. - pub fn name(&self) -> XorName { + /// Returns the xorname. + pub fn xorname(&self) -> XorName { self.address.xorname() } @@ -148,6 +180,16 @@ impl Scratchpad { pub fn payload_size(&self) -> usize { self.encrypted_data.len() } + + /// Size of the scratchpad + pub fn size(&self) -> usize { + size_of::() + self.payload_size() + } + + /// Returns true if the scratchpad is too big + pub fn is_too_big(&self) -> bool { + self.size() > Self::MAX_SIZE + } } #[cfg(test)] @@ -155,11 +197,29 @@ mod tests { use super::*; #[test] - fn test_scratchpad_is_valid() { + fn test_scratchpad_sig_and_update() { let sk = SecretKey::random(); - let pk = sk.public_key(); - let mut scratchpad = Scratchpad::new(pk, 42); - scratchpad.update_and_sign(Bytes::from_static(b"data to be encrypted"), &sk); - assert!(scratchpad.is_valid()); + let raw_data = Bytes::from_static(b"data to be encrypted"); + let mut scratchpad = Scratchpad::new(&sk, 42, &raw_data, 0); + assert!(scratchpad.verify_signature()); + assert_eq!(scratchpad.counter(), 0); + assert_ne!(scratchpad.encrypted_data(), &raw_data); + + let raw_data2 = Bytes::from_static(b"data to be encrypted v2"); + scratchpad.update(&raw_data2, &sk); + assert!(scratchpad.verify_signature()); + assert_eq!(scratchpad.counter(), 1); + assert_ne!(scratchpad.encrypted_data(), &raw_data); + assert_ne!(scratchpad.encrypted_data(), &raw_data2); + } + + #[test] + fn test_scratchpad_encryption() { + let sk = SecretKey::random(); + let raw_data = Bytes::from_static(b"data to be encrypted"); + let scratchpad = Scratchpad::new(&sk, 42, &raw_data, 0); + + let decrypted_data = scratchpad.decrypt_data(&sk).unwrap(); + assert_eq!(decrypted_data, raw_data); } } diff --git a/ant-protocol/src/storage/transaction.rs b/ant-protocol/src/storage/transaction.rs deleted file mode 100644 index 6f7a7a9b11..0000000000 --- a/ant-protocol/src/storage/transaction.rs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use super::address::TransactionAddress; -use bls::SecretKey; -use serde::{Deserialize, Serialize}; - -// re-exports -pub use bls::{PublicKey, Signature}; - -/// Content of a transaction, limited to 32 bytes -pub type TransactionContent = [u8; 32]; - -/// A generic Transaction on the Network -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Ord, PartialOrd)] -pub struct Transaction { - pub owner: PublicKey, - pub parents: Vec, - pub content: TransactionContent, - pub outputs: Vec<(PublicKey, TransactionContent)>, - /// signs the above 4 fields with the owners key - pub signature: Signature, -} - -impl Transaction { - /// Create a new transaction, signing it with the provided secret key. - pub fn new( - owner: PublicKey, - parents: Vec, - content: TransactionContent, - outputs: Vec<(PublicKey, TransactionContent)>, - signing_key: &SecretKey, - ) -> Self { - let signature = signing_key.sign(Self::bytes_to_sign(&owner, &parents, &content, &outputs)); - Self { - owner, - parents, - content, - outputs, - signature, - } - } - - /// Create a new transaction, with the signature already calculated. - pub fn new_with_signature( - owner: PublicKey, - parents: Vec, - content: TransactionContent, - outputs: Vec<(PublicKey, TransactionContent)>, - signature: Signature, - ) -> Self { - Self { - owner, - parents, - content, - outputs, - signature, - } - } - - /// Get the bytes that the signature is calculated from. - pub fn bytes_to_sign( - owner: &PublicKey, - parents: &[PublicKey], - content: &[u8], - outputs: &[(PublicKey, TransactionContent)], - ) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&owner.to_bytes()); - bytes.extend_from_slice("parent".as_bytes()); - bytes.extend_from_slice( - &parents - .iter() - .map(|p| p.to_bytes()) - .collect::>() - .concat(), - ); - bytes.extend_from_slice("content".as_bytes()); - bytes.extend_from_slice(content); - bytes.extend_from_slice("outputs".as_bytes()); - bytes.extend_from_slice( - &outputs - .iter() - .flat_map(|(p, c)| [&p.to_bytes(), c.as_slice()].concat()) - .collect::>(), - ); - bytes - } - - pub fn address(&self) -> TransactionAddress { - TransactionAddress::from_owner(self.owner) - } - - /// Get the bytes that the signature is calculated from. - pub fn bytes_for_signature(&self) -> Vec { - Self::bytes_to_sign(&self.owner, &self.parents, &self.content, &self.outputs) - } - - pub fn verify(&self) -> bool { - self.owner - .verify(&self.signature, self.bytes_for_signature()) - } -} diff --git a/ant-registers/Cargo.toml b/ant-registers/Cargo.toml deleted file mode 100644 index 07771048ce..0000000000 --- a/ant-registers/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -authors = ["MaidSafe Developers "] -description = "Register logic for Autonomi" -edition = "2021" -homepage = "https://maidsafe.net" -license = "GPL-3.0" -name = "ant-registers" -readme = "README.md" -repository = "https://github.com/maidsafe/autonomi" -version = "0.4.7" - -[features] -test-utils = [] - -[dependencies] -bls = { package = "blsttc", version = "8.0.1" } -crdts = { version = "7.3", default-features = false, features = ["merkle"] } -hex = "~0.4.3" -rmp-serde = "1.1.1" -serde = { version = "1.0.133", features = [ "derive", "rc" ]} -thiserror = "1.0.23" -tiny-keccak = "~2.0.2" -xor_name = "5.0.0" - -[dev-dependencies] -rand = { version = "~0.8.5", features = ["small_rng"] } -proptest = { version = "1.0.0" } -eyre = "0.6.8" - -[lints] -workspace = true diff --git a/ant-registers/README.md b/ant-registers/README.md deleted file mode 100644 index c3d87d6813..0000000000 --- a/ant-registers/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# ant-registers - -Provides utilities for working with registers on Autonomi. - -## Introduction to Registers - -Registers are a fundamental data structure in the Safe Network, -designed for storing and managing mutable data with strong consistency guarantees. -They are particularly useful for scenarios requiring atomic updates -and conflict resolution in a distributed environment. - -### General Purpose and Structure - -A register consists of: -- A unique address on the network, determined by its meta and owner - - meta being a user specific string, or a `name` of the register -- `Permissions` showing the who can mutate the register -- An inner CRDT data, which holds the actuall content and the update history - -Registers are: -- Replicated: Stored across multiple nodes for redundancy and availability. -- Versioned: Each update creates a new version, allowing for history tracking. -- Conflict-resistant: Uses a Conflict-free Replicated Data Type (CRDT) approach. - -### API and Workflow - -The `ant-registers` crate provides a high-level API for interacting with registers: - -1. Create a new register -2. Read the current state of a register -3. Write new data to a register -4. Merge register with different versions - -Basic workflow: -1. Initialize a connection to the Safe Network -2. Create or retrieve a register by its address -3. Perform operations (read/write) on the register -4. Handle any conflicts that may arise during concurrent updates - -### Constraints and Limitations - -- Size limits: Individual entry has a maximum size (1024 bytes), - and a register shall have max 1024 entires -- Write permissions: Only authorized owners can modify a register -- Network dependency: Operations require a connection to the Safe Network - -### Understanding MerkleReg in the crdts Crate -1. Purpose of MerkleReg - -MerkleReg is a CRDT that maintains a single value but keeps track of all the changes (mutations) made to that value. -It uses a Merkle tree to store and verify the history of mutations. -This allows for efficient verification of the state of the register and the history of changes, -which is particularly useful in distributed systems where you may need to prove the integrity of data. - -2. Structure of MerkleReg - -The MerkleReg CRDT typically consists of: - * Value: The current value stored in the register. - * History: A Merkle tree that stores the history of all previous values. - Each mutation adds a new node to the tree, which is cryptographically linked to its predecessors, - forming a secure chain of updates. - -3. Mutating the Register - -When you mutate the MerkleReg, the following happens: - * The current value is replaced with the new value. - * The mutation is recorded in the Merkle tree by creating a new node - that includes a cryptographic hash of the new value and the hash of the previous state (root of the Merkle tree). - -4. Conflict Resolution - -Like other CRDTs, MerkleReg resolves conflicts automatically. -If two or more replicas concurrently update the register with different values, -the CRDT framework handles merging these changes. -The Merkle tree structure helps in efficiently reconciling these updates by comparing the histories. - -5. Showing Mutation History in MerkleReg - -To show the mutation history in a MerkleReg, you can traverse the Merkle tree, -listing all previous values and their associated metadata (such as timestamps or versions). -Here’s how you might approach this in practice: -- Traversing the Merkle Tree: - To retrieve the mutation history, you need to walk through the Merkle tree stored in the MerkleReg. - The tree is composed of nodes where each node represents a mutation, containing: - - The value after the mutation. - - A hash that links back to the previous state. -- Displaying the History: - You can then display each value along with its position in the Merkle tree (e.g., the hash or index). - This provides a chronological view of the register’s state over time. - -## Examples - -Here are some simple scenarios using the `ant-registers` crate: - -1. Creating and writing to a register: -```rust -// `permissions` defines the owner of the register -let mut register = Register::new(owner.pub_key, meta, permissions); -let entry = Entry::new("Hello, Safe Network!".as_bytes().to_vec()); -// Children being an empty list for a newly created register -register.write(entry, children, owner.priv_key).await?; -``` - -2. Reading from a register: -```rust -/// Note this reads the root content (i.e. the last entry) of the inner crdt. -/// It will return with multiple `roots`, when there are branches of the inner crdt. -let root_contents = register.read().await?; -for content in root_contents { - println!("content: {:?}", String::from_utf8(content.value)?); -} -``` - -3. Merge registers: -```rust -/// Note two registers are only mergeable when they are for the same address and permissions. -/// And it is the inner crdt to be merged. -pub fn merge(&mut self, other: &Register) -> Result<()> { - self.verify_is_mergeable(other)?; - self.crdt.merge(other.crdt.clone()); - Ok(()) -} -``` diff --git a/ant-registers/src/address.rs b/ant-registers/src/address.rs deleted file mode 100644 index f8f2c346a1..0000000000 --- a/ant-registers/src/address.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::error::{Error, Result}; - -use bls::{PublicKey, PK_SIZE}; -use serde::{Deserialize, Serialize}; -use std::{ - fmt::{Debug, Display}, - hash::Hash, -}; -use xor_name::{XorName, XOR_NAME_LEN}; - -/// Address of a Register on the SAFE Network -#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub struct RegisterAddress { - /// User chosen meta, can be anything, the register's name on the network will be the hash of this meta and the owner - pub(crate) meta: XorName, - /// Owner of the register - pub(crate) owner: PublicKey, -} - -impl Display for RegisterAddress { - /// Display the register address in hex format that can be parsed by `RegisterAddress::from_hex`. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", &self.to_hex()) - } -} - -impl Debug for RegisterAddress { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "RegisterAddress({}) {{ meta: {:?}, owner: {:?} }}", - &self.to_hex()[0..6], - self.meta, - self.owner - ) - } -} - -impl RegisterAddress { - /// Construct a new `RegisterAddress` given `meta` and `owner`. - pub fn new(meta: XorName, owner: PublicKey) -> Self { - Self { meta, owner } - } - - /// Return the network name of the register. - /// This is used to locate the register on the network. - pub fn xorname(&self) -> XorName { - let mut bytes = vec![]; - bytes.extend_from_slice(&self.meta.0); - bytes.extend_from_slice(&self.owner.to_bytes()); - XorName::from_content(&bytes) - } - - /// Serialize this `RegisterAddress` instance to a hex-encoded `String`. - pub fn to_hex(&self) -> String { - let mut bytes = vec![]; - bytes.extend_from_slice(&self.meta.0); - bytes.extend_from_slice(&self.owner.to_bytes()); - hex::encode(bytes) - } - - /// Deserialize a hex-encoded representation of a `RegisterAddress` to a `RegisterAddress` instance. - pub fn from_hex(hex: &str) -> Result { - let bytes = hex::decode(hex).map_err(|_| Error::HexDeserializeFailed)?; - let meta_bytes: [u8; XOR_NAME_LEN] = bytes[..XOR_NAME_LEN] - .try_into() - .map_err(|_| Error::HexDeserializeFailed)?; - let meta = XorName(meta_bytes); - let owner_bytes: [u8; PK_SIZE] = bytes[XOR_NAME_LEN..] - .try_into() - .map_err(|_| Error::HexDeserializeFailed)?; - let owner = PublicKey::from_bytes(owner_bytes).map_err(|_| Error::HexDeserializeFailed)?; - Ok(Self { meta, owner }) - } - - /// Return the user chosen meta. - pub fn meta(&self) -> XorName { - self.meta - } - - /// Return the owner. - pub fn owner(&self) -> PublicKey { - self.owner - } -} - -#[cfg(test)] -mod tests { - use bls::SecretKey; - - use super::*; - - #[test] - fn test_register_hex_conversion() { - let mut rng = rand::thread_rng(); - let owner = SecretKey::random().public_key(); - let meta = XorName::random(&mut rng); - let addr = RegisterAddress::new(meta, owner); - let hex = &addr.to_hex(); - let addr2 = RegisterAddress::from_hex(hex).unwrap(); - - assert_eq!(addr, addr2); - - let bad_hex = format!("{hex}0"); - let err = RegisterAddress::from_hex(&bad_hex); - assert_eq!(err, Err(Error::HexDeserializeFailed)); - } -} diff --git a/ant-registers/src/error.rs b/ant-registers/src/error.rs deleted file mode 100644 index bb4ed37032..0000000000 --- a/ant-registers/src/error.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use bls::PublicKey; -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use crate::{EntryHash, RegisterAddress}; - -#[derive(Error, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub enum Error { - /// Register operation destination address mismatch - #[error( - "The CRDT operation cannot be applied since the Register operation destination address ({dst_addr}) \ - doesn't match the targeted Register's address: {reg_addr}" - )] - RegisterAddrMismatch { - /// Register operation destination address - dst_addr: Box, - /// Targeted Register's address - reg_addr: Box, - }, - /// Entry is too big to fit inside a register - #[error("Entry is too big to fit inside a register: {size}, max: {max}")] - EntryTooBig { - /// Size of the entry - size: usize, - /// Maximum entry size allowed - max: usize, - }, - /// Access denied for user - #[error("Access denied for user: {0:?}")] - AccessDenied(PublicKey), - /// Cannot add another entry since the register entry cap has been reached. - #[error("Cannot add another entry since the register entry cap has been reached: {0}")] - TooManyEntries(usize), - /// Entry could not be found on the data - #[error("Requested entry not found {0}")] - NoSuchEntry(EntryHash), - /// Serialisation Failed - #[error("Serialisation failed")] - SerialisationFailed, - /// SignedRegister Merge only works when both registers have the same base register (owner/permissions/etc) - #[error("SignedRegister Merge failed because base Register was different")] - DifferentBaseRegister, - /// Invalid Signature found in register op - #[error("Invalid signature")] - InvalidSignature, - /// Missing Signature when expecting one in register op - #[error("Missing signature")] - MissingSignature, - /// Signer is not the owner of the Register when attempting to sign a Register - #[error("Invalid SecretKey provided, signer is not the owner of the Register")] - InvalidSecretKey, - /// The register obtained was not the one requested - #[error("Got Register with an invalid register address, requested: {requested}, got: {got}")] - InvalidRegisterAddress { - requested: Box, - got: Box, - }, - /// The provided String can't be deserialized as a RegisterAddress - #[error("Failed to deserialize hex RegisterAddress")] - HexDeserializeFailed, -} - -pub(crate) type Result = std::result::Result; diff --git a/ant-registers/src/metadata.rs b/ant-registers/src/metadata.rs deleted file mode 100644 index 546c2d7f5d..0000000000 --- a/ant-registers/src/metadata.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use serde::{Deserialize, Serialize}; -use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; - -/// An entry in a Register (note that the `vec` is size limited: `MAX_REG_ENTRY_SIZE`) -pub type Entry = Vec; - -/// Hash of the register entry. Logging as the same format of `XorName`. -#[derive(Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct EntryHash(pub crdts::merkle_reg::Hash); - -impl Debug for EntryHash { - fn fmt(&self, formatter: &mut Formatter) -> FmtResult { - write!(formatter, "{self}") - } -} - -impl Display for EntryHash { - fn fmt(&self, formatter: &mut Formatter) -> FmtResult { - write!( - formatter, - "{:02x}{:02x}{:02x}..", - self.0[0], self.0[1], self.0[2] - ) - } -} diff --git a/ant-registers/src/permissions.rs b/ant-registers/src/permissions.rs deleted file mode 100644 index f126155dcf..0000000000 --- a/ant-registers/src/permissions.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use bls::PublicKey; -use serde::{Deserialize, Serialize}; -use std::{collections::BTreeSet, hash::Hash}; - -/// Register permissions -/// Everyone can read a Register, all data is public on safe network. -/// The Default value is nobody can write. -#[derive(Clone, Serialize, Deserialize, PartialEq, PartialOrd, Ord, Eq, Hash, Debug)] -pub enum Permissions { - /// Anyone can write to this Register - AnyoneCanWrite, - /// This is the list of users allowed to write to this Register - Writers(BTreeSet), -} - -impl Default for Permissions { - fn default() -> Permissions { - Permissions::Writers(BTreeSet::default()) - } -} - -impl Permissions { - /// Constructs a new set of permissions with a list of users allowed to write - /// Empty list means nobody can write - pub fn new_with(writers: impl IntoIterator) -> Self { - Self::Writers(writers.into_iter().collect()) - } - - /// Constructs a new set of permissions where everyone can write - pub fn new_anyone_can_write() -> Self { - Self::AnyoneCanWrite - } - - /// Checks is everyone can write to this Register - pub fn can_anyone_write(&self) -> bool { - matches!(self, Self::AnyoneCanWrite) - } - - /// Returns true if the given user can write to this Register - pub fn can_write(&self, user: &PublicKey) -> bool { - match self { - Self::AnyoneCanWrite => true, - Self::Writers(writers) => writers.contains(user), - } - } - - /// If this is restricted to a set of users, add a user to the list of users that can write to this Register - pub fn add_writer(&mut self, user: PublicKey) { - if let Self::Writers(writers) = self { - writers.insert(user); - } - } -} diff --git a/ant-registers/src/reg_crdt.rs b/ant-registers/src/reg_crdt.rs deleted file mode 100644 index f93002aefc..0000000000 --- a/ant-registers/src/reg_crdt.rs +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{error::Result, Entry, EntryHash, Error, RegisterAddress, RegisterOp}; - -use crdts::merkle_reg::Node as MerkleDagEntry; -use crdts::{ - merkle_reg::{Hash as CrdtHash, MerkleReg}, - CmRDT, CvRDT, -}; -use serde::{Deserialize, Serialize}; -use std::{ - collections::{BTreeSet, HashSet}, - fmt::{self, Debug, Display, Formatter}, - hash::Hash, -}; -use xor_name::XorName; - -/// Register data type as a CRDT with Access Control -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd)] -pub struct RegisterCrdt { - /// Address on the network of this piece of data - address: RegisterAddress, - /// CRDT to store the actual data, i.e. the items of the Register. - data: MerkleReg, -} - -impl Display for RegisterCrdt { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "(")?; - for (i, entry) in self.data.read().values().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "<{entry:?}>")?; - } - write!(f, ")") - } -} - -impl RegisterCrdt { - /// Constructs a new '`RegisterCrdtImpl`'. - pub fn new(address: RegisterAddress) -> Self { - Self { - address, - data: MerkleReg::new(), - } - } - - /// Returns the address. - pub fn address(&self) -> &RegisterAddress { - &self.address - } - - /// Merge another register into this one. - pub fn merge(&mut self, other: Self) { - self.data.merge(other.data); - } - - /// Returns total number of items in the register. - pub fn size(&self) -> u64 { - (self.data.num_nodes() + self.data.num_orphans()) as u64 - } - - /// Write a new entry to the `RegisterCrdt`, returning the hash - /// of the entry and the CRDT operation without a signature - pub fn write( - &mut self, - entry: Entry, - children: &BTreeSet, - ) -> Result<(EntryHash, RegisterAddress, MerkleDagEntry)> { - let address = *self.address(); - - let children_array: BTreeSet<[u8; 32]> = children.iter().map(|itr| itr.0).collect(); - let crdt_op = self.data.write(entry, children_array); - self.data.apply(crdt_op.clone()); - let hash = crdt_op.hash(); - - Ok((EntryHash(hash), address, crdt_op)) - } - - /// Apply a remote data CRDT operation to this replica of the `RegisterCrdtImpl`. - pub fn apply_op(&mut self, op: RegisterOp) -> Result<()> { - // Let's first check the op is validly signed. - // Note: Perms and valid sig for the op are checked at the upper Register layer. - - // Check the targeting address is correct - if self.address != op.address { - return Err(Error::RegisterAddrMismatch { - dst_addr: Box::new(op.address), - reg_addr: Box::new(self.address), - }); - } - - // Apply the CRDT operation to the Register - self.data.apply(op.crdt_op); - - Ok(()) - } - - /// Get the entry corresponding to the provided `hash` if it exists. - pub fn get(&self, hash: EntryHash) -> Option<&Entry> { - self.data.node(hash.0).map(|node| &node.value) - } - - /// Read current entries (multiple entries occur on concurrent writes). - pub fn read(&self) -> BTreeSet<(EntryHash, Entry)> { - self.data - .read() - .hashes_and_nodes() - .map(|(hash, node)| (EntryHash(hash), node.value.clone())) - .collect() - } - - /// Returns the children of an entry, along with their corresponding entry hashes - pub fn children(&self, hash: &EntryHash) -> BTreeSet<(EntryHash, Entry)> { - self.data - .children(hash.0) - .hashes_and_nodes() - .map(|(hash, node)| (EntryHash(hash), node.value.clone())) - .collect() - } - - /// Access the underlying MerkleReg (e.g. for access to history) - /// NOTE: This API is unstable and may be removed in the future - pub fn merkle_reg(&self) -> &MerkleReg { - &self.data - } - - /// Log the structure of the MerkleReg as a tree view. - /// This is actually being the `update history` of the register. - pub fn log_update_history(&self) -> String { - let mut output = "MerkleReg Structure:\n".to_string(); - output = format!( - "{output}Total entries: {}\n", - self.data.num_nodes() + self.data.num_orphans() - ); - - // Find root nodes (entries with no parents) - let roots: Vec<_> = self.data.read().hashes().into_iter().collect(); - - // Print the tree starting from each root - for (i, root) in roots.iter().enumerate() { - let mut visited = HashSet::new(); - Self::print_tree( - root, - &self.data, - &mut output, - "", - i == roots.len() - 1, - &mut visited, - ); - } - - output - } - - // Helper function to recursively print the MerkleReg tree - fn print_tree( - hash: &CrdtHash, - merkle_reg: &MerkleReg, - output: &mut String, - prefix: &str, - is_last: bool, - visited: &mut HashSet, - ) { - let pretty_hash = format!("{}", XorName::from_content(hash)); - if !visited.insert(*hash) { - *output = format!( - "{}{prefix}{}* {pretty_hash} (cycle detected)\n", - output, - if is_last { "└── " } else { "├── " }, - ); - return; - } - - let entry = if let Some(node) = merkle_reg.node(*hash) { - format!("value: {}", XorName::from_content(&node.value)) - } else { - "value: None".to_string() - }; - *output = format!( - "{}{prefix}{}{pretty_hash}: {entry}\n", - output, - if is_last { "└── " } else { "├── " }, - ); - - let children: Vec<_> = merkle_reg.children(*hash).hashes().into_iter().collect(); - let new_prefix = format!("{prefix}{} ", if is_last { " " } else { "│" }); - - for (i, child) in children.iter().enumerate() { - Self::print_tree( - child, - merkle_reg, - output, - &new_prefix, - i == children.len() - 1, - visited, - ); - } - - visited.remove(hash); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use bls::SecretKey; - use xor_name::XorName; - - #[test] - fn creating_entry_hash() -> Result<()> { - let mut rng = rand::thread_rng(); - let address_1 = RegisterAddress { - meta: XorName::random(&mut rng), - owner: SecretKey::random().public_key(), - }; - let address_2 = RegisterAddress { - meta: XorName::random(&mut rng), - owner: SecretKey::random().public_key(), - }; - - let mut crdt_1 = RegisterCrdt::new(address_1); - let mut crdt_2 = RegisterCrdt::new(address_2); - let mut parents = BTreeSet::new(); - - let entry_1 = vec![0x1, 0x1]; - // Different RegisterCrdtImpl shall create same hashes for the same entry from root - let (entry_hash_1, _, _) = crdt_1.write(entry_1.clone(), &parents)?; - let (entry_hash_2, _, _) = crdt_2.write(entry_1, &parents)?; - assert!(entry_hash_1 == entry_hash_2); - - let entry_2 = vec![0x2, 0x2]; - // RegisterCrdtImpl shall create different hashes for different entries from root - let (entry_hash_1_2, _, _) = crdt_1.write(entry_2, &parents)?; - assert!(entry_hash_1 != entry_hash_1_2); - - let entry_3 = vec![0x3, 0x3]; - // Different RegisterCrdtImpl shall create same hashes for the same entry from same parents - let _ = parents.insert(entry_hash_1); - let (entry_hash_1_3, _, _) = crdt_1.write(entry_3.clone(), &parents)?; - let (entry_hash_2_3, _, _) = crdt_1.write(entry_3, &parents)?; - assert!(entry_hash_1_3 == entry_hash_2_3); - - Ok(()) - } - - #[test] - fn entry_children() -> Result<()> { - let mut rng = rand::thread_rng(); - let address = RegisterAddress { - meta: XorName::random(&mut rng), - owner: SecretKey::random().public_key(), - }; - let mut crdt = RegisterCrdt::new(address); - - // let's build the following entries hierarchy to test: - // - entry_1 has no child - // - entry_2_1, entry_2_2, and entry_2_3, all have entry_1 as child - // - entry_3 has both entry_2_1 and entry_2_2 as children - let entry_1 = vec![0x0, 0x1]; - let entry_2_1 = vec![0x2, 0x1]; - let entry_2_2 = vec![0x2, 0x2]; - let entry_2_3 = vec![0x2, 0x3]; - let entry_3 = vec![0x0, 0x3]; - let (entry_hash_1, _, _) = crdt.write(entry_1.clone(), &BTreeSet::new())?; - let (entry_hash_2_1, _, _) = - crdt.write(entry_2_1.clone(), &[entry_hash_1].into_iter().collect())?; - let (entry_hash_2_2, _, _) = - crdt.write(entry_2_2.clone(), &[entry_hash_1].into_iter().collect())?; - let (entry_hash_2_3, _, _) = - crdt.write(entry_2_3.clone(), &[entry_hash_1].into_iter().collect())?; - let (entry_hash_3, _, _) = crdt.write( - entry_3, - &[entry_hash_2_1, entry_hash_2_2].into_iter().collect(), - )?; - - let children_entry_1 = crdt.children(&entry_hash_1); - assert_eq!(children_entry_1, BTreeSet::new()); - - let children_entry_2_1 = crdt.children(&entry_hash_2_1); - let children_entry_2_2 = crdt.children(&entry_hash_2_2); - let children_entry_2_3 = crdt.children(&entry_hash_2_3); - assert_eq!( - children_entry_2_1, - [(entry_hash_1, entry_1)].into_iter().collect() - ); - assert_eq!(children_entry_2_1, children_entry_2_2); - assert_eq!(children_entry_2_1, children_entry_2_3); - - let children_entry_3 = crdt.children(&entry_hash_3); - assert_eq!( - children_entry_3, - [(entry_hash_2_1, entry_2_1), (entry_hash_2_2, entry_2_2)] - .into_iter() - .collect() - ); - - Ok(()) - } -} diff --git a/ant-registers/src/register.rs b/ant-registers/src/register.rs deleted file mode 100644 index 2bfda88aa3..0000000000 --- a/ant-registers/src/register.rs +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{error::Result, Error, Permissions, RegisterAddress, RegisterOp}; -#[cfg(feature = "test-utils")] -use bls::SecretKey; -use bls::{PublicKey, Signature}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; -use xor_name::XorName; - -/// Arbitrary maximum size of a register entry. -const MAX_REG_ENTRY_SIZE: usize = 1024; - -/// Maximum number of entries of a register. -const MAX_REG_NUM_ENTRIES: u16 = 1024; - -/// A Register on the SAFE Network -#[derive(Clone, Eq, PartialEq, PartialOrd, Hash, Serialize, Deserialize, Debug)] -pub struct Register { - /// contains the info of meta (XorName) and owner (PublicKey) - address: RegisterAddress, - /// Permissions of the Register - /// Depending on the permissions, the owner can allow other users to write to the register - /// Everyone can always read the Register because all data is public - permissions: Permissions, -} - -/// A Signed Register on the SAFE Network -/// This cryptographically secure version of the Register is used to make sure that the data cannot be tampered with -#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq, Eq, Hash)] -pub struct SignedRegister { - /// the base register we had at creation - register: Register, - /// signature over the above register by the owner - signature: Signature, - /// operations to apply on this register, - /// they contain a signature of the writer - ops: BTreeSet, -} - -impl SignedRegister { - /// Create a new SignedRegister - pub fn new(register: Register, signature: Signature, ops: BTreeSet) -> Self { - Self { - register, - signature, - ops, - } - } - - /// Return the base register. This is the register before any operations have been applied. - pub fn base_register(&self) -> &Register { - &self.register - } - - /// Verfies a SignedRegister - pub fn verify(&self) -> Result<()> { - let reg_size = self.ops.len(); - if reg_size >= MAX_REG_NUM_ENTRIES as usize { - return Err(Error::TooManyEntries(reg_size)); - } - - let bytes = self.register.bytes()?; - if !self - .register - .owner() - .verify(&self.signature, bytes.as_slice()) - { - return Err(Error::InvalidSignature); - } - - for op in &self.ops { - self.register.check_register_op(op)?; - let size = op.crdt_op.value.len(); - if size > MAX_REG_ENTRY_SIZE { - return Err(Error::EntryTooBig { - size, - max: MAX_REG_ENTRY_SIZE, - }); - } - } - Ok(()) - } - - pub fn verify_with_address(&self, address: RegisterAddress) -> Result<()> { - if self.register.address() != &address { - return Err(Error::InvalidRegisterAddress { - requested: Box::new(address), - got: Box::new(*self.address()), - }); - } - self.verify() - } - - /// Merge two SignedRegisters - pub fn merge(&mut self, other: &Self) -> Result<()> { - self.register.verify_is_mergeable(&other.register)?; - self.ops.extend(other.ops.clone()); - Ok(()) - } - - /// Merge two SignedRegisters but verify the incoming content - /// Significantly slower than merge, use when you want to trust but verify the `other` - pub fn verified_merge(&mut self, other: &Self) -> Result<()> { - self.register.verify_is_mergeable(&other.register)?; - other.verify()?; - self.ops.extend(other.ops.clone()); - Ok(()) - } - - /// Return the address. - pub fn address(&self) -> &RegisterAddress { - self.register.address() - } - - /// Return the owner of the data. - pub fn owner(&self) -> PublicKey { - self.register.owner() - } - - /// Check and add an Op to the SignedRegister - pub fn add_op(&mut self, op: RegisterOp) -> Result<()> { - let reg_size = self.ops.len(); - if reg_size >= MAX_REG_NUM_ENTRIES as usize { - return Err(Error::TooManyEntries(reg_size)); - } - - let size = op.crdt_op.value.len(); - if size > MAX_REG_ENTRY_SIZE { - return Err(Error::EntryTooBig { - size, - max: MAX_REG_ENTRY_SIZE, - }); - } - - self.register.check_register_op(&op)?; - self.ops.insert(op); - Ok(()) - } - - /// Returns the reference to the ops list - pub fn ops(&self) -> &BTreeSet { - &self.ops - } - - /// Used in tests. - #[cfg(feature = "test-utils")] - pub fn test_new_from_address(address: RegisterAddress, owner: &SecretKey) -> Self { - let base_register = Register { - address, - permissions: Permissions::AnyoneCanWrite, - }; - let bytes = if let Ok(bytes) = base_register.bytes() { - bytes - } else { - panic!("Failed to serialize register {base_register:?}"); - }; - let signature = owner.sign(bytes); - Self::new(base_register, signature, BTreeSet::new()) - } -} - -impl Register { - /// Create a new Register - pub fn new(owner: PublicKey, meta: XorName, mut permissions: Permissions) -> Self { - permissions.add_writer(owner); - Self { - address: RegisterAddress { meta, owner }, - permissions, - } - } - - /// Returns a bytes version of the Register used for signing - /// Use this API when you want to sign a Register withtout providing a secret key to the Register API - pub fn bytes(&self) -> Result> { - rmp_serde::to_vec(self).map_err(|_| Error::SerialisationFailed) - } - - /// Return the address. - pub fn address(&self) -> &RegisterAddress { - &self.address - } - - /// Return the owner of the data. - pub fn owner(&self) -> PublicKey { - self.address.owner() - } - - /// Return the permission. - pub fn permissions(&self) -> &Permissions { - &self.permissions - } - - /// Check if a register op is valid for our current register - pub fn check_register_op(&self, op: &RegisterOp) -> Result<()> { - if self.permissions.can_anyone_write() { - return Ok(()); // anyone can write, so no need to check the signature - } - self.check_user_permissions(op.source)?; - op.verify_signature(&op.source) - } - - /// Helper to check user write permissions for the given requester's public key. - /// - /// Returns: - /// `Ok(())` if the user can write to this register - /// `Err::AccessDenied` if the user cannot write to this register - pub fn check_user_permissions(&self, requester: PublicKey) -> Result<()> { - if self.permissions.can_write(&requester) { - Ok(()) - } else { - Err(Error::AccessDenied(requester)) - } - } - - // Private helper to check if this Register is mergeable with another - fn verify_is_mergeable(&self, other: &Self) -> Result<()> { - if self.address() != other.address() || self.permissions != other.permissions { - return Err(Error::DifferentBaseRegister); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::{RegisterCrdt, RegisterOp}; - - use super::*; - - use bls::SecretKey; - use rand::{thread_rng, Rng}; - use std::collections::BTreeSet; - use xor_name::XorName; - - #[test] - fn register_create() { - let meta = xor_name::rand::random(); - let (authority_sk, register) = &gen_reg_replicas(None, meta, None, 1)[0]; - - let authority = authority_sk.public_key(); - assert_eq!(register.owner(), authority); - assert_eq!(register.owner(), authority); - - let address = RegisterAddress::new(meta, authority); - assert_eq!(*register.address(), address); - } - - #[test] - fn register_permissions() -> eyre::Result<()> { - let owner_sk = SecretKey::random(); - let owner = owner_sk.public_key(); - let user_sk_1 = SecretKey::random(); - let other_user = user_sk_1.public_key(); - let user_sk_2 = SecretKey::random(); - - let meta: XorName = xor_name::rand::random(); - let address = RegisterAddress { meta, owner }; - - // Create replicas where anyone can write to them, including the owner ofc - let mut signed_reg_1 = create_reg_replica_with( - meta, - Some(owner_sk.clone()), - Some(Permissions::new_anyone_can_write()), - ); - // ...owner and any other users can both write to them - let op = generate_random_op(address, &owner_sk)?; - assert!(signed_reg_1.add_op(op).is_ok()); - let op = generate_random_op(address, &user_sk_1)?; - assert!(signed_reg_1.add_op(op).is_ok()); - let op = generate_random_op(address, &user_sk_2)?; - assert!(signed_reg_1.add_op(op).is_ok()); - - // Create replicas allowing both the owner and other user to write to them - let mut signed_reg_2 = create_reg_replica_with( - meta, - Some(owner_sk.clone()), - Some(Permissions::new_with([other_user])), - ); - // ...owner and the other user can both write to them, others shall fail - let op = generate_random_op(address, &owner_sk)?; - assert!(signed_reg_2.add_op(op).is_ok()); - let op = generate_random_op(address, &user_sk_1)?; - assert!(signed_reg_2.add_op(op).is_ok()); - let op = generate_random_op(address, &user_sk_2)?; - assert!(signed_reg_2.add_op(op).is_err()); - - // Create replicas with the owner as the only allowed to write - let mut signed_reg_3 = create_reg_replica_with(meta, Some(owner_sk.clone()), None); - // ...owner can write to them - let op = generate_random_op(address, &owner_sk)?; - assert!(signed_reg_3.add_op(op).is_ok()); - // ...whilst other user cannot write to them - let op = generate_random_op(address, &user_sk_1)?; - let res = signed_reg_3.add_op(op); - assert!( - matches!(&res, Err(err) if err == &Error::AccessDenied(other_user)), - "Unexpected result: {res:?}" - ); - - // Registers with different permission can not be merged - let res1 = signed_reg_1.merge(&signed_reg_2); - let res2 = signed_reg_2.merge(&signed_reg_1); - assert!( - matches!(&res1, Err(err) if err == &Error::DifferentBaseRegister), - "Unexpected result: {res1:?}" - ); - assert_eq!(res1, res2); - - Ok(()) - } - - #[test] - fn register_query_public_perms() -> eyre::Result<()> { - let meta = xor_name::rand::random(); - - // one register will allow write ops to anyone - let authority_sk1 = SecretKey::random(); - let authority_pk1 = authority_sk1.public_key(); - let owner1 = authority_pk1; - let perms1 = Permissions::new_anyone_can_write(); - let replica1 = create_reg_replica_with(meta, Some(authority_sk1), Some(perms1)); - - // the other register will allow write ops to 'owner1' and 'owner2' only - let authority_sk2 = SecretKey::random(); - let authority_pk2 = authority_sk2.public_key(); - let owner2 = authority_pk2; - let perms2 = Permissions::new_with([owner1]); - let replica2 = create_reg_replica_with(meta, Some(authority_sk2), Some(perms2)); - - // dummy owner - let sk_rand = SecretKey::random(); - let random_user = sk_rand.public_key(); - let sk_rand2 = SecretKey::random(); - let random_user2 = sk_rand2.public_key(); - - // check register 1 is public - assert_eq!(replica1.owner(), authority_pk1); - assert_eq!(replica1.register.check_user_permissions(owner1), Ok(())); - assert_eq!(replica1.register.check_user_permissions(owner2), Ok(())); - assert_eq!( - replica1.register.check_user_permissions(random_user), - Ok(()) - ); - assert_eq!( - replica1.register.check_user_permissions(random_user2), - Ok(()) - ); - - // check register 2 has only owner1 and owner2 write allowed - assert_eq!(replica2.owner(), authority_pk2); - assert_eq!(replica2.register.check_user_permissions(owner1), Ok(())); - assert_eq!(replica2.register.check_user_permissions(owner2), Ok(())); - assert_eq!( - replica2.register.check_user_permissions(random_user), - Err(Error::AccessDenied(random_user)) - ); - assert_eq!( - replica2.register.check_user_permissions(random_user2), - Err(Error::AccessDenied(random_user2)) - ); - - Ok(()) - } - - #[test] - fn exceeding_max_reg_entries_errors() -> eyre::Result<()> { - let meta = xor_name::rand::random(); - - // one replica will allow write ops to anyone - let authority_sk1 = SecretKey::random(); - let owner = authority_sk1.public_key(); - let perms1 = Permissions::new_anyone_can_write(); - let address = RegisterAddress { meta, owner }; - - let mut replica = create_reg_replica_with(meta, Some(authority_sk1.clone()), Some(perms1)); - - for _ in 0..MAX_REG_NUM_ENTRIES { - let op = generate_random_op(address, &authority_sk1)?; - assert!(replica.add_op(op).is_ok()); - } - - let op = generate_random_op(address, &authority_sk1)?; - - let excess_entry = replica.add_op(op); - - match excess_entry { - Err(Error::TooManyEntries(size)) => { - assert_eq!(size, 1024); - Ok(()) - } - anything_else => { - eyre::bail!( - "Expected Excess entries error was not found. Instead: {anything_else:?}" - ) - } - } - } - - // Helpers for tests - fn gen_reg_replicas( - authority_sk: Option, - meta: XorName, - perms: Option, - count: usize, - ) -> Vec<(SecretKey, SignedRegister)> { - let replicas: Vec<(SecretKey, SignedRegister)> = (0..count) - .map(|_| { - let authority_sk = authority_sk.clone().unwrap_or_else(SecretKey::random); - let authority = authority_sk.public_key(); - let perms = perms.clone().unwrap_or_default(); - let register = Register::new(authority, meta, perms); - - let signature = authority_sk.sign(register.bytes().unwrap()); - let signed_reg = SignedRegister::new(register, signature, Default::default()); - - (authority_sk, signed_reg) - }) - .collect(); - - assert_eq!(replicas.len(), count); - replicas - } - - fn create_reg_replica_with( - meta: XorName, - authority_sk: Option, - perms: Option, - ) -> SignedRegister { - let replicas = gen_reg_replicas(authority_sk, meta, perms, 1); - replicas[0].1.clone() - } - - fn random_register_entry() -> Vec { - let random_bytes = thread_rng().gen::<[u8; 32]>(); - random_bytes.to_vec() - } - - fn generate_random_op(address: RegisterAddress, writer_sk: &SecretKey) -> Result { - let mut crdt_reg = RegisterCrdt::new(address); - let item = random_register_entry(); - let (_hash, addr, crdt_op) = crdt_reg.write(item, &BTreeSet::new())?; - Ok(RegisterOp::new(addr, crdt_op, writer_sk)) - } -} diff --git a/ant-registers/src/register_op.rs b/ant-registers/src/register_op.rs deleted file mode 100644 index 455d26b43d..0000000000 --- a/ant-registers/src/register_op.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{error::Result, Entry, Error, RegisterAddress}; - -use bls::{PublicKey, SecretKey}; -use crdts::merkle_reg::Node as MerkleDagEntry; -use serde::{Deserialize, Serialize}; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -/// Register mutation operation to apply to Register. -/// CRDT Data operation applicable to other Register replica. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct RegisterOp { - /// Address of a Register object on the network. - pub(crate) address: RegisterAddress, - /// The data operation to apply. - pub(crate) crdt_op: MerkleDagEntry, - /// The PublicKey of the entity that generated the operation - pub(crate) source: PublicKey, - /// The signature of source on hash(address, crdt_op, source) required to apply the op - pub(crate) signature: bls::Signature, -} - -impl std::hash::Hash for RegisterOp { - fn hash(&self, state: &mut H) { - self.address.hash(state); - self.crdt_op.hash().hash(state); - self.source.hash(state); - self.signature.hash(state); - } -} - -impl RegisterOp { - /// Create a new RegisterOp - pub fn new( - address: RegisterAddress, - crdt_op: MerkleDagEntry, - signer: &SecretKey, - ) -> Self { - let source = signer.public_key(); - let signature = signer.sign(Self::bytes_for_signing(&address, &crdt_op, &source)); - Self { - address, - crdt_op, - source, - signature, - } - } - - /// address of the register this op is destined for - pub fn address(&self) -> RegisterAddress { - self.address - } - - /// the entity that generated the operation - pub fn source(&self) -> PublicKey { - self.source - } - - /// Check signature of register Op against provided public key - pub fn verify_signature(&self, pk: &PublicKey) -> Result<()> { - let bytes = Self::bytes_for_signing(&self.address, &self.crdt_op, &self.source); - if !pk.verify(&self.signature, bytes) { - return Err(Error::InvalidSignature); - } - Ok(()) - } - - /// Returns a bytes version of the RegisterOp used for signing - fn bytes_for_signing( - address: &RegisterAddress, - crdt_op: &MerkleDagEntry, - source: &PublicKey, - ) -> Vec { - let mut hasher = DefaultHasher::new(); - address.hash(&mut hasher); - crdt_op.hash().hash(&mut hasher); - source.hash(&mut hasher); - let hash_value = hasher.finish(); - let bytes = hash_value.to_ne_bytes(); - bytes.to_vec() - } -} diff --git a/ant-service-management/Cargo.toml b/ant-service-management/Cargo.toml index 662634a107..16db7269c0 100644 --- a/ant-service-management/Cargo.toml +++ b/ant-service-management/Cargo.toml @@ -7,16 +7,16 @@ license = "GPL-3.0" name = "ant-service-management" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.4.7" +version = "0.4.8" [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-logging = { path = "../ant-logging", version = "0.2.45" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3", features = ["rpc"] } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0", features = ["rpc"] } async-trait = "0.1" dirs-next = "2.0.0" -libp2p = { version = "0.54.1", features = ["kad"] } +libp2p = { version = "0.55.0", features = ["kad"] } libp2p-identity = { version = "0.2.7", features = ["rand"] } prost = { version = "0.9" } serde = { version = "1.0", features = ["derive"] } diff --git a/ant-service-management/src/node.rs b/ant-service-management/src/node.rs index 3c281ba4b7..cd92f6bac0 100644 --- a/ant-service-management/src/node.rs +++ b/ant-service-management/src/node.rs @@ -110,11 +110,6 @@ impl ServiceStateActions for NodeService<'_> { args.push(OsString::from(max_log_files.to_string())); } - if let Some(owner) = &self.service_data.owner { - args.push(OsString::from("--owner")); - args.push(OsString::from(owner)); - } - args.push(OsString::from("--rewards-address")); args.push(OsString::from( self.service_data.rewards_address.to_string(), @@ -288,8 +283,6 @@ pub struct NodeServiceData { pub max_log_files: Option, #[serde(default)] pub metrics_port: Option, - #[serde(default)] - pub owner: Option, pub network_id: Option, #[serde(default)] pub node_ip: Option, diff --git a/ant-token-supplies/Cargo.toml b/ant-token-supplies/Cargo.toml index 66a3661079..c788f13d04 100644 --- a/ant-token-supplies/Cargo.toml +++ b/ant-token-supplies/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "ant-token-supplies" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.62" +version = "0.1.63" [dependencies] diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml index 31ea027dd5..b00e6d3459 100644 --- a/autonomi/Cargo.toml +++ b/autonomi/Cargo.toml @@ -3,7 +3,7 @@ authors = ["MaidSafe Developers "] description = "Autonomi client API" name = "autonomi" license = "GPL-3.0" -version = "0.3.5" +version = "0.3.6" edition = "2021" homepage = "https://maidsafe.net" readme = "README.md" @@ -15,78 +15,55 @@ crate-type = ["cdylib", "rlib"] [[example]] name = "data_and_archive" -required-features = ["full"] [[example]] name = "put_and_dir_upload" -required-features = ["full"] [features] -default = ["vault"] +default = [] external-signer = ["ant-evm/external-signer"] -extension-module = ["pyo3/extension-module"] -fs = ["tokio/fs"] -full = ["registers", "vault", "fs"] -local = ["ant-networking/local", "ant-evm/local"] +extension-module = ["pyo3/extension-module", "pyo3-async-runtimes"] loud = [] -registers = [] -vault = ["registers"] [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-networking = { path = "../ant-networking", version = "0.3.4" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } -ant-registers = { path = "../ant-registers", version = "0.4.7" } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-networking = { path = "../ant-networking", version = "0.3.5" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } bip39 = "2.0.0" blst = "0.3.13" blstrs = "0.7.1" bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } const-hex = "1.12.0" -curv = { version = "0.10.1", package = "sn_curv", default-features = false, features = [ - "num-bigint", -] } -eip2333 = { version = "0.2.1", package = "sn_bls_ckd" } +eyre = "0.6.5" futures = "0.3.30" hex = "~0.4.3" -libp2p = "0.54.1" -pyo3 = { version = "0.20", optional = true, features = ["extension-module", "abi3-py38"] } +libp2p = "0.55.0" +pyo3 = { version = "0.23.4", optional = true, features = ["extension-module", "abi3-py38"] } +pyo3-async-runtimes = { version = "0.23", optional = true, features = ["tokio-runtime"] } rand = "0.8.5" +rayon = "1.8.0" rmp-serde = "1.1.1" self_encryption = "~0.30.0" serde = { version = "1.0.133", features = ["derive", "rc"] } -serde-wasm-bindgen = "0.6.5" sha2 = "0.10.6" thiserror = "1.0.23" -tokio = { version = "1.35.0", features = ["sync"] } +tokio = { version = "1.35.0", features = ["sync", "fs"] } tracing = { version = "~0.1.26" } walkdir = "2.5.0" -wasm-bindgen = "0.2.93" -wasm-bindgen-futures = "0.4.43" xor_name = "5.0.0" [dev-dependencies] alloy = { version = "0.7.3", default-features = false, features = ["contract", "json-rpc", "network", "node-bindings", "provider-http", "reqwest-rustls-tls", "rpc-client", "rpc-types", "signer-local", "std"] } -ant-logging = { path = "../ant-logging", version = "0.2.45" } +ant-logging = { path = "../ant-logging", version = "0.2.46" } eyre = "0.6.5" +serial_test = "3.2.0" sha2 = "0.10.6" # Do not specify the version field. Release process expects even the local dev deps to be published. # Removing the version field is a workaround. test-utils = { path = "../test-utils" } -tiny_http = "0.11" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -wasm-bindgen-test = "0.3.43" - -[target.'cfg(target_arch = "wasm32")'.dependencies] -console_error_panic_hook = "0.1.7" -evmlib = { path = "../evmlib", version = "0.1.8", features = ["wasm-bindgen"] } -# See https://github.com/sebcrozet/instant/blob/7bd13f51f5c930239fddc0476a837870fb239ed7/README.md#using-instant-for-a-wasm-platform-where-performancenow-is-not-available -instant = { version = "0.1", features = ["wasm-bindgen", "inaccurate"] } -js-sys = "0.3.70" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing-web = "0.1.3" -xor_name = { version = "5.0.0", features = ["serialize-hex"] } [lints] workspace = true @@ -94,3 +71,5 @@ workspace = true [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] +# Adds snippets from the `examples` dir to items if relevant +cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"] diff --git a/autonomi/README.md b/autonomi/README.md index d77c38a81b..82ce268378 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -30,7 +30,7 @@ async fn main() -> Result<(), Box> { let data_addr = client .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get_public(data_addr).await?; + let _data_fetched = client.data_get_public(&data_addr).await?; // Put and fetch directory from local file system. let dir_addr = client.dir_and_archive_upload_public("files/to/upload".into(), &wallet).await?; @@ -51,10 +51,6 @@ let wallet = Wallet::new_from_private_key(EvmNetwork::ArbitrumSepolia, key)?; let wallet = Wallet::new_from_private_key(EvmNetwork::new_custom("", "", ""), key)?; ``` -# Registers - -Registers are deprecated and planned to be replaced by transactions and pointers. Currently, transactions can already be used. For example usage, see [the transaction test](tests/transaction.rs). Pointers are not yet implemented, but will follow soon. - ## Running tests To run the tests, we can run a local network: @@ -66,30 +62,30 @@ To run the tests, we can run a local network: cargo run --bin evm-testnet ``` -2. Run a local network with the `local` feature and use the local EVM node. +2. Run a local network and use the local EVM node. ```sh - cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-local + cargo run --bin antctl -- local run --build --clean --rewards-address evm-local ``` -3. Then run the tests with the `local` feature and pass the EVM params again: +3. Then run the tests and pass the EVM params again: ```sh - EVM_NETWORK=local cargo test --features local --package autonomi + EVM_NETWORK=local cargo test --package autonomi ``` ### Using a live testnet or mainnet Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and point it to a live network. -1. Run a local network with the `local` feature: +1. Run a local network: ```sh -cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-arbitrum-one +cargo run --bin antctl -- local run --build --clean --rewards-address evm-arbitrum-one ``` 2. Then pass the private key of the wallet, and ensure it has enough gas and payment tokens on the network (in this case Arbitrum One): ```sh -EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package autonomi --features local +EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package autonomi ``` ## Using funds from the Deployer Wallet @@ -135,10 +131,6 @@ Deployer wallet private key: 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efca Genesis wallet balance: (tokens: 20000000000000000000000000, gas: 9998998011366954730202) ``` -# WASM - -For documentation on WASM, see [./README_WASM.md]. - # Python For documentation on the Python bindings, see [./README_PYTHON.md]. diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 6772ce14a1..e359798b25 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -1,14 +1,17 @@ -## Python Bindings +# Autonomi Python Bindings The Autonomi client library provides Python bindings for easy integration with Python applications. -### Installation +## Installation + +We recommend using `uv` for Python environment management: ```bash -pip install autonomi-client +uv venv +uv pip install autonomi-client ``` -### Quick Start +## Quick Start ```python from autonomi_client import Client, Wallet, PaymentOption @@ -34,155 +37,179 @@ retrieved = client.data_get_public(addr) print(f"Retrieved: {retrieved.decode()}") ``` -### Available Modules +## API Reference -#### Core Components +### Client -- `Client`: Main interface to the Autonomi network - - `connect(peers: List[str])`: Connect to network nodes - - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data - - `data_get_public(addr: str)`: Download data - - `data_put(data: bytes, payment: PaymentOption)`: Store private data - - `data_get(access: DataMapChunk)`: Retrieve private data - - `register_generate_key()`: Generate register key +The main interface to interact with the Autonomi network. -- `Wallet`: Ethereum wallet management - - `new(private_key: str)`: Create wallet from private key - - `address()`: Get wallet address - - `balance()`: Get current balance +#### Connection Methods -- `PaymentOption`: Payment configuration - - `wallet(wallet: Wallet)`: Create payment option from wallet +- `connect(peers: List[str]) -> Client` + - Connect to network nodes + - `peers`: List of multiaddresses for initial network nodes -#### Private Data +#### Data Operations -- `DataMapChunk`: Handle private data storage - - `from_hex(hex: str)`: Create from hex string - - `to_hex()`: Convert to hex string - - `address()`: Get short reference address +- `data_put_public(data: bytes, payment: PaymentOption) -> str` + - Upload public data to the network + - Returns address where data is stored -```python -# Private data example -access = client.data_put(secret_data, payment) -print(f"Private data stored at: {access.to_hex()}") -retrieved = client.data_get(access) -``` +- `data_get_public(addr: str) -> bytes` + - Download public data from the network + - `addr`: Address returned from `data_put_public` -#### Registers +- `data_put(data: bytes, payment: PaymentOption) -> DataMapChunk` + - Store private (encrypted) data + - Returns access information for later retrieval -- Register operations for mutable data - - `register_create(value: bytes, name: str, key: RegisterSecretKey, wallet: Wallet)` - - `register_get(address: str)` - - `register_update(register: Register, value: bytes, key: RegisterSecretKey)` +- `data_get(access: DataMapChunk) -> bytes` + - Retrieve private data + - `access`: DataMapChunk from previous `data_put` -```python -# Register example -key = client.register_generate_key() -register = client.register_create(b"Initial value", "my_register", key, wallet) -client.register_update(register, b"New value", key) -``` +#### Pointer Operations -#### Vaults +- `pointer_get(address: str) -> Pointer` + - Retrieve pointer from network + - `address`: Hex-encoded pointer address -- `VaultSecretKey`: Manage vault access - - `new()`: Generate new key - - `from_hex(hex: str)`: Create from hex string - - `to_hex()`: Convert to hex string +- `pointer_put(pointer: Pointer, wallet: Wallet)` + - Store pointer on network + - Requires payment via wallet -- `UserData`: User data management - - `new()`: Create new user data - - `add_file_archive(archive: str)`: Add file archive - - `add_private_file_archive(archive: str)`: Add private archive - - `file_archives()`: List archives - - `private_file_archives()`: List private archives +- `pointer_cost(key: VaultSecretKey) -> str` + - Calculate pointer storage cost + - Returns cost in atto tokens -```python -# Vault example -vault_key = VaultSecretKey.new() -cost = client.vault_cost(vault_key) -client.write_bytes_to_vault(data, payment, vault_key, content_type=1) -data, content_type = client.fetch_and_decrypt_vault(vault_key) -``` +#### Vault Operations -#### Utility Functions +- `vault_cost(key: VaultSecretKey) -> str` + - Calculate vault storage cost -- `encrypt(data: bytes)`: Self-encrypt data -- `hash_to_short_string(input: str)`: Generate short reference +- `write_bytes_to_vault(data: bytes, payment: PaymentOption, key: VaultSecretKey, content_type: int) -> str` + - Write data to vault + - Returns vault address -### Complete Examples +- `fetch_and_decrypt_vault(key: VaultSecretKey) -> Tuple[bytes, int]` + - Retrieve vault data + - Returns (data, content_type) -#### Data Management +- `get_user_data_from_vault(key: VaultSecretKey) -> UserData` + - Get user data from vault -```python -def handle_data_operations(client, payment): - # Upload text - text_data = b"Hello, Safe Network!" - text_addr = client.data_put_public(text_data, payment) - - # Upload binary data - with open("image.jpg", "rb") as f: - image_data = f.read() - image_addr = client.data_put_public(image_data, payment) - - # Download and verify - downloaded = client.data_get_public(text_addr) - assert downloaded == text_data -``` +- `put_user_data_to_vault(key: VaultSecretKey, payment: PaymentOption, user_data: UserData) -> str` + - Store user data in vault + - Returns vault address -#### Private Data and Encryption +### Wallet -```python -def handle_private_data(client, payment): - # Create and encrypt private data - secret = {"api_key": "secret_key"} - data = json.dumps(secret).encode() - - # Store privately - access = client.data_put(data, payment) - print(f"Access token: {access.to_hex()}") - - # Retrieve - retrieved = client.data_get(access) - secret = json.loads(retrieved.decode()) -``` +Ethereum wallet management for payments. -#### Vault Management +- `new(private_key: str) -> Wallet` + - Create wallet from private key + - `private_key`: 64-char hex string without '0x' prefix -```python -def handle_vault(client, payment): - # Create vault - vault_key = VaultSecretKey.new() - - # Store user data - user_data = UserData() - user_data.add_file_archive("archive_address") - - # Save to vault - cost = client.put_user_data_to_vault(vault_key, payment, user_data) - - # Retrieve - retrieved = client.get_user_data_from_vault(vault_key) - archives = retrieved.file_archives() -``` +- `address() -> str` + - Get wallet's Ethereum address -### Error Handling +- `balance() -> str` + - Get wallet's token balance -All operations can raise exceptions. It's recommended to use try-except blocks: +- `balance_of_gas() -> str` + - Get wallet's gas balance -```python -try: - client = Client.connect(peers) - # ... operations ... -except Exception as e: - print(f"Error: {e}") -``` +### PaymentOption + +Configure payment methods. + +- `wallet(wallet: Wallet) -> PaymentOption` + - Create payment option from wallet + +### Pointer + +Handle network pointers for referencing data. + +- `new(target: str) -> Pointer` + - Create new pointer + - `target`: Hex-encoded target address + +- `address() -> str` + - Get pointer's network address + +- `target() -> str` + - Get pointer's target address + +### VaultSecretKey + +Manage vault access keys. + +- `new() -> VaultSecretKey` + - Generate new key + +- `from_hex(hex: str) -> VaultSecretKey` + - Create from hex string + +- `to_hex() -> str` + - Convert to hex string + +### UserData + +Manage user data in vaults. + +- `new() -> UserData` + - Create new user data + +- `add_file_archive(archive: str) -> Optional[str]` + - Add file archive + - Returns archive ID if successful + +- `add_private_file_archive(archive: str) -> Optional[str]` + - Add private archive + - Returns archive ID if successful + +- `file_archives() -> List[Tuple[str, str]]` + - List archives as (id, address) pairs + +- `private_file_archives() -> List[Tuple[str, str]]` + - List private archives as (id, address) pairs + +### DataMapChunk + +Handle private data storage references. + +- `from_hex(hex: str) -> DataMapChunk` + - Create from hex string + +- `to_hex() -> str` + - Convert to hex string + +- `address() -> str` + - Get short reference address + +### Utility Functions + +- `encrypt(data: bytes) -> Tuple[bytes, List[bytes]]` + - Self-encrypt data + - Returns (data_map, chunks) + +## Examples + +See the `examples/` directory for complete examples: +- `autonomi_example.py`: Basic data operations +- `autonomi_pointers.py`: Working with pointers +- `autonomi_vault.py`: Vault operations +- `autonomi_private_data.py`: Private data handling +- `autonomi_private_encryption.py`: Data encryption +- `autonomi_advanced.py`: Advanced usage scenarios -### Best Practices +## Best Practices -1. Always keep private keys secure -2. Use error handling for all network operations -3. Clean up resources when done +1. Always handle wallet private keys securely +2. Check operation costs before executing +3. Use appropriate error handling 4. Monitor wallet balance for payments 5. Use appropriate content types for vault storage +6. Consider using pointers for updatable references +7. Properly manage and backup vault keys -For more examples, see the `examples/` directory in the repository. +For more examples and detailed usage, see the examples in the repository. diff --git a/autonomi/README_WASM.md b/autonomi/README_WASM.md deleted file mode 100644 index 8c6478def7..0000000000 --- a/autonomi/README_WASM.md +++ /dev/null @@ -1,95 +0,0 @@ -# Autonomi JS API - -Note: the JS API is experimental and will be subject to change. - -The entry point for connecting to the network is {@link Client.connect}. - -This API is a wrapper around the Rust API, found here: https://docs.rs/autonomi/latest/autonomi. The Rust API contains more detailed documentation on concepts and some types. - -## Addresses - -For addresses (chunk, data, archives, etc) we're using hex-encoded strings containing a 256-bit XOR addresse. For example: `abcdefg012345678900000000000000000000000000000000000000000000000`. - -## Example - -Note: `getEvmNetwork` will use hardcoded EVM network values that should be set during compilation of this library. - -```javascript -import init, { Client, Wallet, getEvmNetwork } from 'autonomi'; - -let client = await new Client(["/ip4/127.0.0.1/tcp/36075/ws/p2p/12D3KooWALb...BhDAfJY"]); -console.log("connected"); - -let wallet = Wallet.new_from_private_key(getEvmNetwork, "your_private_key_here"); -console.log("wallet retrieved"); - -let data = new Uint8Array([1, 2, 3]); -let result = await client.put(data, wallet); -console.log("Data stored at:", result); - -let fetchedData = await client.get(result); -console.log("Data retrieved:", fetchedData); -``` - -## Funded wallet from custom local network - -```js -const evmNetwork = getEvmNetworkCustom("http://localhost:4343", "", ""); -const wallet = getFundedWalletWithCustomNetwork(evmNetwork, "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"); -``` - -# Developing - -## WebAssembly - -To run a WASM test - -- Install `wasm-pack` -- Make sure your Rust supports the `wasm32-unknown-unknown` target. (If you - have `rustup`: `rustup target add wasm32-unknown-unknown`.) -- Pass a bootstrap peer via `ANT_PEERS`. This *has* to be the websocket address, - e.g. `/ip4//tcp//ws/p2p/`. - - As well as the other environment variables needed for EVM payments (e.g. `RPC_URL`). -- Optionally specify the specific test, e.g. `-- put` to run `put()` in `wasm.rs` only. - -Example: - -```sh -ANT_PEERS=/ip4//tcp//ws/p2p/ wasm-pack test --release --firefox autonomi --features=files --test wasm -- put -``` - -### Test from JS in the browser - -`wasm-pack test` does not execute JavaScript, but runs mostly WebAssembly. Again make sure the environment variables are -set and build the JS package: - -```sh -wasm-pack build --dev --target web autonomi --features=vault -``` - -Then cd into `autonomi/tests-js`, and use `npm` to install and serve the test html file. - -``` -cd autonomi/tests-js -npm install -npm run serve -``` - -Then go to `http://127.0.0.1:8080/tests-js` in the browser. Here, enter a `ws` multiaddr of a local node and press ' -run'. - -## MetaMask example - -There is a MetaMask example for doing a simple put operation. - -Build the package with the `external-signer` feature (and again with the env variables) and run a webserver, e.g. with -Python: - -```sh -wasm-pack build --dev --target web autonomi --features=external-signer -python -m http.server --directory autonomi 8000 -``` - -Then visit `http://127.0.0.1:8000/examples/metamask` in your (modern) browser. - -Here, enter a `ws` multiaddr of a local node and press 'run'. diff --git a/autonomi/examples/data_and_archive.rs b/autonomi/examples/data_and_archive.rs index 07fddd560f..6072d563dd 100644 --- a/autonomi/examples/data_and_archive.rs +++ b/autonomi/examples/data_and_archive.rs @@ -1,4 +1,5 @@ -use autonomi::{Bytes, Client, Metadata, PrivateArchive}; +use autonomi::files::{Metadata, PrivateArchive}; +use autonomi::{Bytes, Client}; use test_utils::evm::get_funded_wallet; use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; @@ -14,8 +15,8 @@ async fn main() -> eyre::Result<()> { // Upload 10MiB of random data and verify it by fetching it back. let data = Bytes::from("Hello, World!"); - let data_map = client.data_put(data.clone(), (&wallet).into()).await?; - let data_fetched = client.data_get(data_map.clone()).await?; + let (_cost, data_map) = client.data_put(data.clone(), (&wallet).into()).await?; + let data_fetched = client.data_get(&data_map).await?; assert_eq!(data, data_fetched); // Upload the data as part of an archive, giving it the name `test.txt`. @@ -27,8 +28,8 @@ async fn main() -> eyre::Result<()> { ); // Upload the archive to the network. - let archive_data_map = client.archive_put(&archive, (&wallet).into()).await?; - let archive_fetched = client.archive_get(archive_data_map).await?; + let (_cost, archive_data_map) = client.archive_put(&archive, (&wallet).into()).await?; + let archive_fetched = client.archive_get(&archive_data_map).await?; assert_eq!(archive, archive_fetched); println!("Archive uploaded successfully"); diff --git a/autonomi/examples/metamask/index.html b/autonomi/examples/metamask/index.html deleted file mode 100644 index 128273acbc..0000000000 --- a/autonomi/examples/metamask/index.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/autonomi/examples/metamask/index.js b/autonomi/examples/metamask/index.js deleted file mode 100644 index 66bf524037..0000000000 --- a/autonomi/examples/metamask/index.js +++ /dev/null @@ -1,233 +0,0 @@ -import init, * as autonomi from '../../pkg/autonomi.js'; - -export async function externalSignerPrivateDataPutToVault(peerAddr) { - try { - // Check if MetaMask (window.ethereum) is available - if (typeof window.ethereum === 'undefined') { - throw new Error('MetaMask is not installed'); - } - - // Request account access from MetaMask - const accounts = await window.ethereum.request({method: 'eth_requestAccounts'}); - const sender = accounts[0]; // Get the first account - - // Setup API client - await init(); - - autonomi.logInit("autonomi=trace"); - - const client = await autonomi.Client.connect([peerAddr]); - - // Generate 1MB of random bytes in a Uint8Array - const data = new Uint8Array(1024 * 1024).map(() => Math.floor(Math.random() * 256)); - - // Encrypt the data to chunks - const [dataMapChunk, dataChunks, dataMapChunkAddress, dataChunkAddresses] = autonomi.encryptData(data); - - // Fetch quotes for the chunks - const [quotes, quotePayments, _freeChunks] = await client.getQuotes(dataChunkAddresses); - - // Pay for data chunks (not the data map) - const receipt = await executeQuotePayments(sender, quotes, quotePayments); - - // Wait for a few seconds to allow tx to confirm - await new Promise(resolve => setTimeout(resolve, 5000)); - - // Upload the data - const privateDataAccess = await client.putPrivateDataWithReceipt(data, receipt); - - // Create a private archive - const privateArchive = new autonomi.PrivateArchive(); - - // Add our data's data map chunk to the private archive - privateArchive.addFile("test", privateDataAccess, autonomi.createMetadata(data.length)); - - // Get the private archive's bytes - const privateArchiveBytes = privateArchive.bytes(); - - // Encrypt the private archive to chunks - const [paDataMapChunk, paDataChunks, paDataMapChunkAddress, paDataChunkAddresses] = autonomi.encryptData(privateArchiveBytes); - - // Fetch quotes for the private archive chunks - const [paQuotes, paQuotePayments, _paFreeChunks] = await client.getQuotes(paDataChunkAddresses); - - // Pay for the private archive chunks (not the data map) - const paReceipt = await executeQuotePayments(sender, paQuotes, paQuotePayments); - - // Wait for a few seconds to allow tx to confirm - await new Promise(resolve => setTimeout(resolve, 5000)); - - // Upload the private archive - const privateArchiveAccess = await client.putPrivateArchiveWithReceipt(privateArchive, paReceipt); - - // Generate a random vault key (should normally be derived from a constant signature) - const vaultKey = autonomi.genSecretKey(); - - // Fetch user data from vault (won't exist, so will be empty) - let userData; - - try { - userData = await client.getUserDataFromVault(vaultKey); - } catch (err) { - userData = new autonomi.UserData(); - } - - // Add archive to user data - userData.addPrivateFileArchive(privateArchiveAccess, "test-archive"); - - // Get or create a scratchpad for the user data - let scratchpad = await client.getOrCreateUserDataScratchpad(vaultKey); - - // Content address of the scratchpad - let scratchPadAddress = scratchpad.xorName(); - - // Fetch quotes for the scratchpad - const [spQuotes, spQuotePayments, _spFreeChunks] = await client.getQuotes(scratchPadAddress ? [scratchPadAddress] : []); - - // Pay for the private archive chunks (not the data map) - const spReceipt = await executeQuotePayments(sender, spQuotes, spQuotePayments); - - // Wait for a few seconds to allow tx to confirm - await new Promise(resolve => setTimeout(resolve, 5000)); - - // Update vault - await client.putUserDataToVaultWithReceipt(userData, spReceipt, vaultKey); - - // VERIFY UPLOADED DATA - - // Fetch user data - let fetchedUserData = await client.getUserDataFromVault(vaultKey); - - // Get the first key - let fetchedPrivateArchiveAccess = fetchedUserData.privateFileArchives().keys().next().value; - - // Get private archive - let fetchedPrivateArchive = await client.getPrivateArchive(fetchedPrivateArchiveAccess); - - // Select first file in private archive - let [fetchedFilePath, [fetchedPrivateFileAccess, fetchedFileMetadata]] = fetchedPrivateArchive.map().entries().next().value; - - console.log(fetchedFilePath); - console.log(fetchedPrivateFileAccess); - console.log(fetchedFileMetadata); - - // Fetch private file/data - let fetchedPrivateFile = await client.getPrivateData(fetchedPrivateFileAccess); - - // Compare to original data - console.log("Comparing fetched data to original data.."); - - if (fetchedPrivateFile.toString() === data.toString()) { - console.log("Data matches! Private file upload to vault was successful!"); - } else { - console.log("Data does not match!! Something went wrong..") - } - } catch (error) { - console.error("An error occurred:", error); - } -} - -// Helper function to send a transaction through MetaMask using Ethereum JSON-RPC -async function sendTransaction({from, to, data}) { - const transactionParams = { - from: from, // Sender address - to: to, // Destination address - data: data, // Calldata (transaction input) - }; - - try { - // Send the transaction via MetaMask and get the transaction hash - const txHash = await window.ethereum.request({ - method: 'eth_sendTransaction', - params: [transactionParams] - }); - - console.log(`Transaction sent with hash: ${txHash}`); - return txHash; // Return the transaction hash - - } catch (error) { - console.error("Failed to send transaction:", error); - throw error; - } -} - -async function waitForTransactionConfirmation(txHash) { - const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)); - - // Poll for the transaction receipt - while (true) { - // Query the transaction receipt - const receipt = await window.ethereum.request({ - method: 'eth_getTransactionReceipt', - params: [txHash], - }); - - // If the receipt is found, the transaction has been mined - if (receipt !== null) { - // Check if the transaction was successful (status is '0x1') - if (receipt.status === '0x1') { - console.log('Transaction successful!', receipt); - return receipt; // Return the transaction receipt - } else { - console.log('Transaction failed!', receipt); - throw new Error('Transaction failed'); - } - } - - // Wait for 1 second before checking again - await delay(1000); - } -} - -const executeQuotePayments = async (sender, quotes, quotePayments) => { - // Get the EVM network - let evmNetwork = autonomi.getEvmNetwork(); - - // Form quotes payment calldata - const payForQuotesCalldata = autonomi.getPayForQuotesCalldata( - evmNetwork, - quotePayments - ); - - // Form approve to spend tokens calldata - const approveCalldata = autonomi.getApproveToSpendTokensCalldata( - evmNetwork, - payForQuotesCalldata.approve_spender, - payForQuotesCalldata.approve_amount - ); - - console.log("Sending approve transaction.."); - - // Approve to spend tokens - let hash = await sendTransaction({ - from: sender, - to: approveCalldata[1], - data: approveCalldata[0] - }); - - // Wait for approve tx to confirm - await waitForTransactionConfirmation(hash); - - let payments = {}; - - // Execute batched quote payment transactions - for (const [calldata, quoteHashes] of payForQuotesCalldata.batched_calldata_map) { - console.log("Sending batched data payment transaction.."); - - let hash = await sendTransaction({ - from: sender, - to: payForQuotesCalldata.to, - data: calldata - }); - - await waitForTransactionConfirmation(hash); - - // Record the transaction hashes for each quote - quoteHashes.forEach(quoteHash => { - payments[quoteHash] = hash; - }); - } - - // Generate receipt - return autonomi.getReceiptFromQuotesAndPayments(quotes, payments); -} \ No newline at end of file diff --git a/autonomi/examples/put_and_dir_upload.rs b/autonomi/examples/put_and_dir_upload.rs index 4af5e20b11..55ede2d89a 100644 --- a/autonomi/examples/put_and_dir_upload.rs +++ b/autonomi/examples/put_and_dir_upload.rs @@ -13,17 +13,17 @@ async fn main() -> Result<(), Box> { let wallet = get_funded_wallet(); // Put and fetch data. - let data_addr = client + let (_cost, data_addr) = client .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get_public(data_addr).await?; + let _data_fetched = client.data_get_public(&data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client + let (_cost, dir_addr) = client .dir_and_archive_upload_public("files/to/upload".into(), &wallet) .await?; client - .dir_download_public(dir_addr, "files/downloaded".into()) + .dir_download_public(&dir_addr, "files/downloaded".into()) .await?; Ok(()) diff --git a/autonomi/nodejs/dist/graphEntry.d.ts b/autonomi/nodejs/dist/graphEntry.d.ts new file mode 100644 index 0000000000..cf5edea7fb --- /dev/null +++ b/autonomi/nodejs/dist/graphEntry.d.ts @@ -0,0 +1,9 @@ +import { GraphEntryOptions, PaymentOption } from './types'; +export declare class GraphEntry { + private nativeList; + private constructor(); + static create(address: string): Promise; + get(): Promise; + put(options: GraphEntryOptions, payment: PaymentOption): Promise; + getCost(key: string): Promise; +} diff --git a/autonomi/nodejs/dist/graphEntry.js b/autonomi/nodejs/dist/graphEntry.js new file mode 100644 index 0000000000..b4dff59764 --- /dev/null +++ b/autonomi/nodejs/dist/graphEntry.js @@ -0,0 +1,25 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GraphEntry = void 0; +class GraphEntry { + constructor(nativeList) { + this.nativeList = nativeList; + } + static async create(address) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async get() { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async put(options, payment) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getCost(key) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } +} +exports.GraphEntry = GraphEntry; diff --git a/autonomi/nodejs/dist/pointer.d.ts b/autonomi/nodejs/dist/pointer.d.ts new file mode 100644 index 0000000000..59b0a16d65 --- /dev/null +++ b/autonomi/nodejs/dist/pointer.d.ts @@ -0,0 +1,9 @@ +import { PointerOptions, PaymentOption } from './types'; +export declare class Pointer { + private nativePointer; + private constructor(); + static create(address: string): Promise; + get(): Promise; + put(options: PointerOptions, payment: PaymentOption): Promise; + getCost(key: string): Promise; +} diff --git a/autonomi/nodejs/dist/pointer.js b/autonomi/nodejs/dist/pointer.js new file mode 100644 index 0000000000..b2161c57aa --- /dev/null +++ b/autonomi/nodejs/dist/pointer.js @@ -0,0 +1,25 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Pointer = void 0; +class Pointer { + constructor(nativePointer) { + this.nativePointer = nativePointer; + } + static async create(address) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async get() { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async put(options, payment) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getCost(key) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } +} +exports.Pointer = Pointer; diff --git a/autonomi/nodejs/dist/vault.d.ts b/autonomi/nodejs/dist/vault.d.ts new file mode 100644 index 0000000000..36f5dcf20e --- /dev/null +++ b/autonomi/nodejs/dist/vault.d.ts @@ -0,0 +1,11 @@ +import { VaultOptions, PaymentOption, UserData } from './types'; +export declare class Vault { + private nativeVault; + private constructor(); + static create(address: string): Promise; + getCost(key: string): Promise; + writeBytes(data: Buffer, payment: PaymentOption, options: VaultOptions): Promise; + fetchAndDecrypt(key: string): Promise<[Buffer, number]>; + getUserData(key: string): Promise; + putUserData(key: string, payment: PaymentOption, userData: UserData): Promise; +} diff --git a/autonomi/nodejs/dist/vault.js b/autonomi/nodejs/dist/vault.js new file mode 100644 index 0000000000..688f6ce42f --- /dev/null +++ b/autonomi/nodejs/dist/vault.js @@ -0,0 +1,33 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Vault = void 0; +class Vault { + constructor(nativeVault) { + this.nativeVault = nativeVault; + } + static async create(address) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getCost(key) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async writeBytes(data, payment, options) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async fetchAndDecrypt(key) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getUserData(key) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async putUserData(key, payment, userData) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } +} +exports.Vault = Vault; diff --git a/autonomi/nodejs/dist/wallet.d.ts b/autonomi/nodejs/dist/wallet.d.ts new file mode 100644 index 0000000000..17c5072a9b --- /dev/null +++ b/autonomi/nodejs/dist/wallet.d.ts @@ -0,0 +1,13 @@ +import { NetworkConfig } from './types'; +export interface WalletConfig { + privateKey?: string; + address?: string; +} +export declare class Wallet { + private nativeWallet; + private constructor(); + static create(config: NetworkConfig & WalletConfig): Promise; + getAddress(): Promise; + getBalance(): Promise; + signMessage(message: string): Promise; +} diff --git a/autonomi/nodejs/dist/wallet.js b/autonomi/nodejs/dist/wallet.js new file mode 100644 index 0000000000..9fa9575fb6 --- /dev/null +++ b/autonomi/nodejs/dist/wallet.js @@ -0,0 +1,25 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Wallet = void 0; +class Wallet { + constructor(nativeWallet) { + this.nativeWallet = nativeWallet; + } + static async create(config) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getAddress() { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async getBalance() { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + async signMessage(message) { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } +} +exports.Wallet = Wallet; diff --git a/autonomi/pyproject.toml b/autonomi/pyproject.toml index 0a17202968..b3c9a2d080 100644 --- a/autonomi/pyproject.toml +++ b/autonomi/pyproject.toml @@ -2,33 +2,50 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" -[tool.maturin] -features = ["extension-module"] -python-source = "python" -module-name = "autonomi_client.autonomi_client" -bindings = "pyo3" -target-dir = "target/wheels" - [project] name = "autonomi-client" dynamic = ["version"] description = "Autonomi client API" -readme = "README.md" +authors = [{ name = "MaidSafe Developers", email = "dev@maidsafe.net" }] +dependencies = ["maturin>=1.7.4", "pip>=24.0"] +readme = "README_PYTHON.md" requires-python = ">=3.8" license = { text = "GPL-3.0" } -keywords = ["safe", "network", "autonomi"] -authors = [{ name = "MaidSafe Developers", email = "dev@maidsafe.net" }] classifiers = [ - "Programming Language :: Python", - "Programming Language :: Python :: Implementation :: CPython", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Rust", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: Implementation :: CPython", ] -dependencies = [ - "pip>=24.3.1", + +[project.urls] +Homepage = "https://maidsafe.net" +Repository = "https://github.com/maidsafe/autonomi" + +[tool.maturin] +features = ["extension-module"] +module-name = "autonomi_client" +python-source = "python" +bindings = "pyo3" +include = ["README_PYTHON.md", "src/*", "python/*", "pyproject.toml"] +manifest-path = "Cargo.toml" +sdist-include = [ + "README_PYTHON.md", + "src/**/*", + "python/**/*", + "pyproject.toml", + "Cargo.toml", ] +workspace = false + +[tool.pytest.ini_options] +testpaths = ["tests/python"] +python_files = ["test_*.py"] +addopts = "-v -s" diff --git a/autonomi/python/autonomi_client/__init__.py b/autonomi/python/autonomi_client/__init__.py index b1e437b894..17b37e7e5c 100644 --- a/autonomi/python/autonomi_client/__init__.py +++ b/autonomi/python/autonomi_client/__init__.py @@ -1,11 +1 @@ -from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt - -__all__ = [ - "Client", - "Wallet", - "PaymentOption", - "VaultSecretKey", - "UserData", - "DataMapChunk", - "encrypt" -] +from .autonomi_client import * diff --git a/autonomi/python/examples/autonomi_data_registers.py b/autonomi/python/examples/autonomi_data_registers.py deleted file mode 100644 index 4d258fefa1..0000000000 --- a/autonomi/python/examples/autonomi_data_registers.py +++ /dev/null @@ -1,89 +0,0 @@ -from autonomi_client import Client, Wallet, PaymentOption, RegisterSecretKey -import hashlib - -def handle_data_operations(client: Client, payment: PaymentOption): - """Example of various data operations""" - print("\n=== Data Operations ===") - - # Upload some text data - text_data = b"Hello, Safe Network!" - text_addr = client.data_put_public(text_data, payment) - print(f"Text data uploaded to: {text_addr}") - - # Upload binary data (like an image) - with open("example.jpg", "rb") as f: - image_data = f.read() - image_addr = client.data_put_public(image_data, payment) - print(f"Image uploaded to: {image_addr}") - - # Download and verify data - downloaded_text = client.data_get_public(text_addr) - assert downloaded_text == text_data, "Text data verification failed!" - print("Text data verified successfully") - - # Download and save image - downloaded_image = client.data_get_public(image_addr) - with open("downloaded_example.jpg", "wb") as f: - f.write(downloaded_image) - print("Image downloaded successfully") - -def handle_register_operations(client: Client, wallet: Wallet): - """Example of register operations""" - print("\n=== Register Operations ===") - - # Create a register key - register_key = client.register_generate_key() - print(f"Generated register key") - - # Create a register with initial value - register_name = "my_first_register" - initial_value = b"Initial register value" - register = client.register_create( - initial_value, - register_name, - register_key, - wallet - ) - print(f"Created register at: {register.address()}") - - # Read current value - values = register.values() - print(f"Current register values: {[v.decode() for v in values]}") - - # Update register value - new_value = b"Updated register value" - client.register_update(register, new_value, register_key) - print("Register updated") - - # Read updated value - updated_register = client.register_get(register.address()) - updated_values = updated_register.values() - print(f"Updated register values: {[v.decode() for v in updated_values]}") - -def main(): - # Initialize wallet and client - private_key = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" - peers = ["/ip4/127.0.0.1/tcp/12000"] - - try: - # Setup - wallet = Wallet(private_key) - print(f"Wallet address: {wallet.address()}") - print(f"Wallet balance: {wallet.balance()}") - - client = Client.connect(peers) - payment = PaymentOption.wallet(wallet) - - # Run examples - handle_data_operations(client, payment) - handle_register_operations(client, wallet) - - except Exception as e: - print(f"Error: {e}") - return 1 - - print("\nAll operations completed successfully!") - return 0 - -if __name__ == "__main__": - exit(main()) \ No newline at end of file diff --git a/autonomi/python/examples/autonomi_example.py b/autonomi/python/examples/autonomi_example.py index 14d6bbfc0e..c889dd5477 100644 --- a/autonomi/python/examples/autonomi_example.py +++ b/autonomi/python/examples/autonomi_example.py @@ -1,6 +1,7 @@ from autonomi_client import Client, Wallet, PaymentOption +import asyncio -def main(): +async def main(): # Initialize a wallet with a private key # This should be a valid Ethereum private key (64 hex chars without '0x' prefix) private_key = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" @@ -9,23 +10,18 @@ def main(): print(f"Wallet balance: {wallet.balance()}") # Connect to the network - # These should be valid multiaddresses of network nodes - peers = [ - "/ip4/127.0.0.1/tcp/12000", - "/ip4/127.0.0.1/tcp/12001" - ] - client = Client.connect(peers) + client = await Client.init() # Create payment option using the wallet payment = PaymentOption.wallet(wallet) # Upload some data data = b"Hello, Safe Network!" - addr = client.data_put_public(data, payment) + addr = await client.data_put_public(data, payment) print(f"Data uploaded to address: {addr}") # Download the data back - downloaded = client.data_get_public(addr) + downloaded = await client.data_get_public(addr) print(f"Downloaded data: {downloaded.decode()}") # You can also upload files @@ -34,5 +30,5 @@ def main(): file_addr = client.data_put_public(file_data, payment) print(f"File uploaded to address: {file_addr}") -if __name__ == "__main__": - main() \ No newline at end of file + +asyncio.run(main()) diff --git a/autonomi/python/examples/autonomi_pointers.py b/autonomi/python/examples/autonomi_pointers.py new file mode 100644 index 0000000000..fb63ec4451 --- /dev/null +++ b/autonomi/python/examples/autonomi_pointers.py @@ -0,0 +1,56 @@ +""" +Example demonstrating the use of pointers in the Autonomi network. +Pointers allow for creating references to data that can be updated. +""" + +from autonomi_client import Client, Network, Wallet, PaymentOption, SecretKey, PointerTarget, ChunkAddress, Pointer +import asyncio + +async def main(): + # Initialize a wallet with a private key + # This should be a valid Ethereum private key (64 hex chars) + private_key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + network = Network(True) + wallet = Wallet.new_from_private_key(network, private_key) + print(f"Wallet address: {wallet.address()}") + print(f"Wallet balance: {await wallet.balance()}") + + # Connect to the network + client = await Client.init_local() + + # First, let's upload some data that we want to point to + target_data = b"Hello, I'm the target data!" + target_addr = await client.data_put_public(target_data, PaymentOption.wallet(wallet)) + print(f"Target data uploaded to: {target_addr}") + + # Create a pointer target from the address + target = PointerTarget.from_chunk_address(ChunkAddress(target_addr)) + + # Create owner key pair + key = SecretKey() + + # Estimate the cost of the pointer + cost = await client.pointer_cost(key.public_key()) + print(f"pointer cost: {cost}") + + # Create the pointer + pointer = Pointer(key, 0, target) + payment_option = PaymentOption.wallet(wallet) + + # Create and store the pointer + pointer_addr = await client.pointer_put(pointer, payment_option) + print("Pointer stored successfully") + + # Wait for the pointer to be stored by the network + await asyncio.sleep(1) + + # Later, we can retrieve the pointer + pointer = await client.pointer_get(pointer_addr) + print(f"Retrieved pointer target: {pointer}") + + # We can then use the target address to get the original data + retrieved_data = await client.data_get_public(pointer.target.hex) + print(f"Retrieved target data: {retrieved_data.decode()}") + + +asyncio.run(main()) diff --git a/autonomi/python/examples/autonomi_private_data.py b/autonomi/python/examples/autonomi_private_data.py index 4d68acd3ea..bf33ba9ee4 100644 --- a/autonomi/python/examples/autonomi_private_data.py +++ b/autonomi/python/examples/autonomi_private_data.py @@ -1,4 +1,4 @@ -from autonomi_client import Client, Wallet, PaymentOption, RegisterSecretKey, RegisterPermissions +from autonomi_client import Client, Wallet, PaymentOption from typing import List, Optional import json @@ -16,24 +16,6 @@ def store_private_data(self, data: bytes) -> str: def retrieve_private_data(self, addr: str) -> bytes: """Retrieve privately stored data""" return self.client.data_get(addr) - - def create_shared_register(self, name: str, initial_value: bytes, - allowed_writers: List[str]) -> str: - """Create a register that multiple users can write to""" - register_key = self.client.register_generate_key() - - # Create permissions for all writers - permissions = RegisterPermissions.new_with(allowed_writers) - - register = self.client.register_create_with_permissions( - initial_value, - name, - register_key, - permissions, - self.wallet - ) - - return register.address() def main(): # Initialize @@ -62,23 +44,6 @@ def main(): retrieved_json = json.loads(retrieved_data.decode()) print(f"Retrieved data: {retrieved_json}") - # Create shared register - allowed_writers = [ - wallet.address(), # self - "0x1234567890abcdef1234567890abcdef12345678" # another user - ] - register_addr = manager.create_shared_register( - "shared_config", - b"initial shared data", - allowed_writers - ) - print(f"Created shared register at: {register_addr}") - - # Verify register - register = client.register_get(register_addr) - values = register.values() - print(f"Register values: {[v.decode() for v in values]}") - except Exception as e: print(f"Error: {e}") return 1 diff --git a/autonomi/python/examples/basic.py b/autonomi/python/examples/basic.py index 4ddaee182c..38ee9abebd 100644 --- a/autonomi/python/examples/basic.py +++ b/autonomi/python/examples/basic.py @@ -1,4 +1,4 @@ -from autonomi_client import Client, Wallet, RegisterSecretKey, VaultSecretKey, UserData +from autonomi_client import Client, Wallet, VaultSecretKey, UserData def external_signer_example(client: Client, data: bytes): # Get quotes for storing data @@ -33,12 +33,6 @@ def main(): private_data = client.data_get(private_access) print(f"Retrieved private data: {private_data}") - # Create register - reg_addr = client.register_create(b"Initial value", "my_register", wallet) - print(f"Created register at: {reg_addr}") - reg_values = client.register_get(reg_addr) - print(f"Register values: {reg_values}") - # Upload file/directory file_addr = client.file_upload_public("./test_data", wallet) print(f"Uploaded files to: {file_addr}") diff --git a/autonomi/setup.py b/autonomi/setup.py new file mode 100644 index 0000000000..f7d5530dd0 --- /dev/null +++ b/autonomi/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup +from setuptools_rust import RustExtension + +setup( + name="autonomi-client", + version="0.3.0", + description="Autonomi client API", + long_description=open("README_PYTHON.md").read(), + long_description_content_type="text/markdown", + author="MaidSafe Developers", + author_email="dev@maidsafe.net", + url="https://github.com/maidsafe/autonomi", + rust_extensions=[ + RustExtension( + "autonomi_client.autonomi_client", + "Cargo.toml", + features=["extension-module"], + py_limited_api=True, + debug=False, + ) + ], + packages=["autonomi_client"], + package_dir={"": "python"}, + zip_safe=False, + python_requires=">=3.8", + classifiers=[ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + ], +) \ No newline at end of file diff --git a/autonomi/src/client/config.rs b/autonomi/src/client/config.rs new file mode 100644 index 0000000000..41b73e29c0 --- /dev/null +++ b/autonomi/src/client/config.rs @@ -0,0 +1,192 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_evm::EvmNetwork; +use ant_networking::{GetRecordCfg, PutRecordCfg, VerificationKind}; +use ant_protocol::messages::ChunkProof; +use libp2p::{kad::Record, Multiaddr, PeerId}; +use rand::{thread_rng, Rng}; +use std::{collections::HashSet, num::NonZero}; + +pub use ant_networking::{ResponseQuorum, RetryStrategy}; + +/// Configuration for the [`crate::Client`] which can be provided through: [`crate::Client::init_with_config`]. +#[derive(Debug, Clone, Default)] +pub struct ClientConfig { + /// Whether we're expected to connect to a local network. + pub local: bool, + + /// List of peers to connect to. + /// + /// If not provided, the client will use the default bootstrap peers. + pub peers: Option>, + + /// EVM network to use for quotations and payments. + pub evm_network: EvmNetwork, + + /// Strategy for data operations by the client. + pub strategy: ClientOperatingStrategy, +} + +impl ClientConfig { + pub fn local(peers: Option>) -> Self { + Self { + local: true, + peers, + evm_network: EvmNetwork::new(true).unwrap_or_default(), + strategy: Default::default(), + } + } +} + +/// Strategy configuration for data operations by the client. +/// +/// Default values are used for each type of data, but you can override them here. +#[derive(Debug, Clone)] +pub struct ClientOperatingStrategy { + pub chunks: Strategy, + pub graph_entry: Strategy, + pub pointer: Strategy, + pub scratchpad: Strategy, +} + +impl ClientOperatingStrategy { + pub fn new() -> Self { + Default::default() + } +} + +/// The default configuration for the client. +/// +/// It is optimized for faster chunk put and get, benefiting from the chunk content addressed property. +/// Other data types are optimized for fast verification, and resilience in case of forks, which are impossible for chunks. +impl Default for ClientOperatingStrategy { + fn default() -> Self { + let two = NonZero::new(2).expect("2 is non 0"); + Self { + chunks: Strategy { + put_quorum: ResponseQuorum::N(two), + put_retry: RetryStrategy::Balanced, + verification_quorum: ResponseQuorum::N(two), + verification_retry: RetryStrategy::Balanced, + get_quorum: ResponseQuorum::One, // chunks are content addressed so one is enough as there is no fork possible + get_retry: RetryStrategy::Quick, + verification_kind: VerificationKind::Network, // it is recommended to use [`Strategy::chunk_put_cfg`] for chunks to benefit from the chunk proof + }, + graph_entry: Strategy { + put_quorum: ResponseQuorum::Majority, + put_retry: RetryStrategy::Balanced, + verification_quorum: ResponseQuorum::Majority, + verification_retry: RetryStrategy::Quick, // verification should be quick + get_quorum: ResponseQuorum::N(two), // forks are rare but possible, balance between resilience and speed + get_retry: RetryStrategy::Quick, + verification_kind: VerificationKind::Crdt, // forks are possible + }, + pointer: Strategy { + put_quorum: ResponseQuorum::Majority, + put_retry: RetryStrategy::Balanced, + verification_quorum: ResponseQuorum::Majority, + verification_retry: RetryStrategy::Quick, // verification should be quick + get_quorum: ResponseQuorum::Majority, // majority to catch possible differences in versions + get_retry: RetryStrategy::Quick, + verification_kind: VerificationKind::Crdt, // forks are possible + }, + scratchpad: Strategy { + put_quorum: ResponseQuorum::Majority, + put_retry: RetryStrategy::Balanced, + verification_quorum: ResponseQuorum::Majority, + verification_retry: RetryStrategy::Quick, // verification should be quick + get_quorum: ResponseQuorum::Majority, // majority to catch possible differences in versions + get_retry: RetryStrategy::Quick, + verification_kind: VerificationKind::Crdt, // forks are possible + }, + } + } +} + +/// The strategy to adopt when puting and getting data from the network +/// +/// Puts are followed by a verification using get, to ensure the data is stored correctly. This verification can be configured separately from the regular gets. +#[derive(Debug, Clone)] +pub struct Strategy { + /// The number of responses to wait for before considering the put operation successful + pub put_quorum: ResponseQuorum, + /// The retry strategy to use if we fail to store a piece of data + pub put_retry: RetryStrategy, + /// The number of responses to wait for before considering the verification to be successful + pub verification_quorum: ResponseQuorum, + /// The retry strategy for verification + pub verification_retry: RetryStrategy, + /// The number of responses to wait for before considering the get operation successful + pub get_quorum: ResponseQuorum, + /// The retry strategy to use if the get operation fails + pub get_retry: RetryStrategy, + /// Verification kind + pub(crate) verification_kind: VerificationKind, +} + +impl Strategy { + /// Get config for getting a record + pub(crate) fn get_cfg(&self) -> GetRecordCfg { + GetRecordCfg { + get_quorum: self.get_quorum, + retry_strategy: self.get_retry, + target_record: None, + expected_holders: HashSet::new(), + } + } + + /// Get config for verifying the existance of a record + pub(crate) fn verification_cfg(&self) -> GetRecordCfg { + GetRecordCfg { + get_quorum: self.verification_quorum, + retry_strategy: self.verification_retry, + target_record: None, + expected_holders: HashSet::new(), + } + } + + /// Put config for storing a record + pub(crate) fn put_cfg(&self, put_to: Option>) -> PutRecordCfg { + PutRecordCfg { + put_quorum: self.put_quorum, + retry_strategy: self.put_retry, + use_put_record_to: put_to, + verification: Some((self.verification_kind.clone(), self.verification_cfg())), + } + } + + /// Put config for storing a Chunk, more strict and requires a chunk proof of storage + pub(crate) fn chunk_put_cfg(&self, expected: Record, put_to: Vec) -> PutRecordCfg { + let random_nonce = thread_rng().gen::(); + let expected_proof = ChunkProof::new(&expected.value, random_nonce); + + PutRecordCfg { + put_quorum: self.put_quorum, + retry_strategy: self.put_retry, + use_put_record_to: Some(put_to), + verification: Some(( + VerificationKind::ChunkProof { + expected_proof, + nonce: random_nonce, + }, + self.verification_cfg_specific(expected), + )), + } + } + + /// Get config for verifying the existance and value of a record + pub(crate) fn verification_cfg_specific(&self, expected: Record) -> GetRecordCfg { + GetRecordCfg { + get_quorum: self.verification_quorum, + retry_strategy: self.verification_retry, + target_record: Some(expected), + expected_holders: HashSet::new(), + } + } +} diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs deleted file mode 100644 index e64c6872e4..0000000000 --- a/autonomi/src/client/data/mod.rs +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::sync::LazyLock; - -use ant_evm::{Amount, EvmWalletError}; -use ant_networking::NetworkError; -use ant_protocol::storage::Chunk; -use ant_protocol::NetworkAddress; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use xor_name::XorName; - -use crate::client::payment::PaymentOption; -use crate::client::{ClientEvent, UploadSummary}; -use crate::{self_encryption::encrypt, Client}; - -pub mod public; - -/// Number of chunks to upload in parallel. -/// -/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. -pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk upload batch size: {}", batch_size); - batch_size -}); - -/// Number of chunks to download in parallel. -/// -/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk download batch size: {}", batch_size); - batch_size -}); - -/// Number of retries to upload chunks. -pub(crate) const RETRY_ATTEMPTS: usize = 3; - -/// Raw Data Address (points to a DataMap) -pub type DataAddr = XorName; -/// Raw Chunk Address (points to a [`Chunk`]) -pub type ChunkAddr = XorName; - -/// Errors that can occur during the put operation. -#[derive(Debug, thiserror::Error)] -pub enum PutError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("A network error occurred.")] - Network(#[from] NetworkError), - #[error("Error occurred during cost estimation.")] - CostError(#[from] CostError), - #[error("Error occurred during payment.")] - PayError(#[from] PayError), - #[error("Serialization error: {0}")] - Serialization(String), - #[error("A wallet error occurred.")] - Wallet(#[from] ant_evm::EvmError), - #[error("The vault owner key does not match the client's public key")] - VaultBadOwner, - #[error("Payment unexpectedly invalid for {0:?}")] - PaymentUnexpectedlyInvalid(NetworkAddress), - #[error("The payment proof contains no payees.")] - PayeesMissing, -} - -/// Errors that can occur during the pay operation. -#[derive(Debug, thiserror::Error)] -pub enum PayError { - #[error("Wallet error: {0:?}")] - EvmWalletError(#[from] EvmWalletError), - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Cost error: {0:?}")] - Cost(#[from] CostError), -} - -/// Errors that can occur during the get operation. -#[derive(Debug, thiserror::Error)] -pub enum GetError { - #[error("Could not deserialize data map.")] - InvalidDataMap(rmp_serde::decode::Error), - #[error("Failed to decrypt data.")] - Decryption(crate::self_encryption::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), - #[error("General networking error: {0:?}")] - Network(#[from] NetworkError), - #[error("General protocol error: {0:?}")] - Protocol(#[from] ant_protocol::Error), -} - -/// Errors that can occur during the cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum CostError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Could not get store quote for: {0:?} after several retries")] - CouldNotGetStoreQuote(XorName), - #[error("Could not get store costs: {0:?}")] - CouldNotGetStoreCosts(NetworkError), - #[error("Not enough node quotes for {0:?}, got: {1:?} and need at least {2:?}")] - NotEnoughNodeQuotes(XorName, usize, usize), - #[error("Failed to serialize {0}")] - Serialization(String), - #[error("Market price error: {0:?}")] - MarketPriceError(#[from] ant_evm::payment_vault::error::Error), -} - -/// Private data on the network can be accessed with this -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct DataMapChunk(Chunk); - -impl DataMapChunk { - pub fn to_hex(&self) -> String { - hex::encode(self.0.value()) - } - - pub fn from_hex(hex: &str) -> Result { - let data = hex::decode(hex)?; - Ok(Self(Chunk::new(Bytes::from(data)))) - } - - /// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side. - pub fn address(&self) -> String { - hash_to_short_string(&self.to_hex()) - } -} - -fn hash_to_short_string(input: &str) -> String { - let mut hasher = DefaultHasher::new(); - input.hash(&mut hasher); - let hash_value = hasher.finish(); - hash_value.to_string() -} - -impl Client { - /// Fetch a blob of (private) data from the network - /// - /// # Example - /// - /// ```no_run - /// use autonomi::{Client, Bytes}; - /// # #[tokio::main] - /// # async fn main() -> Result<(), Box> { - /// # let client = Client::init().await?; - /// # let data_map = todo!(); - /// let data_fetched = client.data_get(data_map).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn data_get(&self, data_map: DataMapChunk) -> Result { - info!( - "Fetching private data from Data Map {:?}", - data_map.0.address() - ); - let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; - - debug!("Successfully fetched a blob of private data from the network"); - Ok(data) - } - - /// Upload a piece of private data to the network. This data will be self-encrypted. - /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. - /// - /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. - /// - /// # Example - /// - /// ```no_run - /// use autonomi::{Client, Bytes}; - /// # #[tokio::main] - /// # async fn main() -> Result<(), Box> { - /// # let client = Client::init().await?; - /// # let wallet = todo!(); - /// let data = Bytes::from("Hello, World"); - /// let data_map = client.data_put(data, wallet).await?; - /// let data_fetched = client.data_get(data_map).await?; - /// assert_eq!(data, data_fetched); - /// # Ok(()) - /// # } - /// ``` - pub async fn data_put( - &self, - data: Bytes, - payment_option: PaymentOption, - ) -> Result { - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, chunks) = encrypt(data)?; - debug!("Encryption took: {:.2?}", now.elapsed()); - - // Pay for all chunks - let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); - info!("Paying for {} addresses", xor_names.len()); - let receipt = self - .pay_for_content_addrs(xor_names.into_iter(), payment_option) - .await - .inspect_err(|err| error!("Error paying for data: {err:?}"))?; - - // Upload the chunks with the payments - debug!("Uploading {} chunks", chunks.len()); - - let mut failed_uploads = self - .upload_chunks_with_retries(chunks.iter().collect(), &receipt) - .await; - - // Return the last chunk upload error - if let Some(last_chunk_fail) = failed_uploads.pop() { - tracing::error!( - "Error uploading chunk ({:?}): {:?}", - last_chunk_fail.0.address(), - last_chunk_fail.1 - ); - return Err(last_chunk_fail.1); - } - - let record_count = chunks.len(); - - // Reporting - if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = receipt - .values() - .map(|(_, cost)| cost.as_atto()) - .sum::(); - - let summary = UploadSummary { - record_count, - tokens_spent, - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err:?}"); - } - } - - Ok(DataMapChunk(data_map_chunk)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hex() { - let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello"))); - let hex = data_map.to_hex(); - let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex"); - assert_eq!(data_map, data_map2); - } -} diff --git a/autonomi/src/client/data/public.rs b/autonomi/src/client/data/public.rs deleted file mode 100644 index edac6df18f..0000000000 --- a/autonomi/src/client/data/public.rs +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use bytes::Bytes; -use libp2p::kad::Quorum; -use std::collections::HashSet; - -use crate::client::payment::{PaymentOption, Receipt}; -use crate::client::utils::process_tasks_with_max_concurrency; -use crate::client::{ClientEvent, UploadSummary}; -use crate::{self_encryption::encrypt, Client}; -use ant_evm::{Amount, AttoTokens}; -use ant_networking::{GetRecordCfg, NetworkError}; -use ant_protocol::{ - storage::{ - try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind, RetryStrategy, - }, - NetworkAddress, -}; - -use super::*; - -impl Client { - /// Fetch a blob of data from the network - pub async fn data_get_public(&self, addr: DataAddr) -> Result { - info!("Fetching data from Data Address: {addr:?}"); - let data_map_chunk = self.chunk_get(addr).await?; - let data = self - .fetch_from_data_map_chunk(data_map_chunk.value()) - .await?; - - debug!("Successfully fetched a blob of data from the network"); - Ok(data) - } - - /// Upload a piece of data to the network. - /// Returns the Data Address at which the data was stored. - /// This data is publicly accessible. - pub async fn data_put_public( - &self, - data: Bytes, - payment_option: PaymentOption, - ) -> Result { - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, chunks) = encrypt(data)?; - let data_map_addr = data_map_chunk.address(); - debug!("Encryption took: {:.2?}", now.elapsed()); - info!("Uploading datamap chunk to the network at: {data_map_addr:?}"); - - let map_xor_name = *data_map_chunk.address().xorname(); - let mut xor_names = vec![map_xor_name]; - - for chunk in &chunks { - xor_names.push(*chunk.name()); - } - - // Pay for all chunks + data map chunk - info!("Paying for {} addresses", xor_names.len()); - let receipt = self - .pay_for_content_addrs(xor_names.into_iter(), payment_option) - .await - .inspect_err(|err| error!("Error paying for data: {err:?}"))?; - - // Upload all the chunks in parallel including the data map chunk - debug!("Uploading {} chunks", chunks.len()); - - let mut failed_uploads = self - .upload_chunks_with_retries( - chunks - .iter() - .chain(std::iter::once(&data_map_chunk)) - .collect(), - &receipt, - ) - .await; - - // Return the last chunk upload error - if let Some(last_chunk_fail) = failed_uploads.pop() { - tracing::error!( - "Error uploading chunk ({:?}): {:?}", - last_chunk_fail.0.address(), - last_chunk_fail.1 - ); - return Err(last_chunk_fail.1); - } - - let record_count = chunks.len() + 1; - - // Reporting - if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = receipt - .values() - .map(|(_proof, price)| price.as_atto()) - .sum::(); - - let summary = UploadSummary { - record_count, - tokens_spent, - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err:?}"); - } - } - - Ok(map_xor_name) - } - - /// Get a raw chunk from the network. - pub async fn chunk_get(&self, addr: ChunkAddr) -> Result { - info!("Getting chunk: {addr:?}"); - - let key = NetworkAddress::from_chunk_address(ChunkAddress::new(addr)).to_record_key(); - debug!("Fetching chunk from network at: {key:?}"); - let get_cfg = GetRecordCfg { - get_quorum: Quorum::One, - retry_strategy: Some(RetryStrategy::Balanced), - target_record: None, - expected_holders: HashSet::new(), - is_register: false, - }; - - let record = self - .network - .get_record_from_network(key, &get_cfg) - .await - .inspect_err(|err| error!("Error fetching chunk: {err:?}"))?; - let header = RecordHeader::from_record(&record)?; - - if let RecordKind::Chunk = header.kind { - let chunk: Chunk = try_deserialize_record(&record)?; - Ok(chunk) - } else { - error!( - "Record kind mismatch: expected Chunk, got {:?}", - header.kind - ); - Err(NetworkError::RecordKindMismatch(RecordKind::Chunk).into()) - } - } - - /// Get the estimated cost of storing a piece of data. - pub async fn data_cost(&self, data: Bytes) -> Result { - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, chunks) = encrypt(data)?; - - debug!("Encryption took: {:.2?}", now.elapsed()); - - let map_xor_name = *data_map_chunk.address().xorname(); - let mut content_addrs = vec![map_xor_name]; - - for chunk in &chunks { - content_addrs.push(*chunk.name()); - } - - info!( - "Calculating cost of storing {} chunks. Data map chunk at: {map_xor_name:?}", - content_addrs.len() - ); - - let store_quote = self - .get_store_quotes(content_addrs.into_iter()) - .await - .inspect_err(|err| error!("Error getting store quotes: {err:?}"))?; - - let total_cost = AttoTokens::from_atto( - store_quote - .0 - .values() - .map(|quote| quote.price()) - .sum::(), - ); - - Ok(total_cost) - } - - // Upload chunks and retry failed uploads up to `RETRY_ATTEMPTS` times. - pub(crate) async fn upload_chunks_with_retries<'a>( - &self, - mut chunks: Vec<&'a Chunk>, - receipt: &Receipt, - ) -> Vec<(&'a Chunk, PutError)> { - let mut current_attempt: usize = 1; - - loop { - let mut upload_tasks = vec![]; - for chunk in chunks { - let self_clone = self.clone(); - let address = *chunk.address(); - - let Some((proof, _)) = receipt.get(chunk.name()) else { - debug!("Chunk at {address:?} was already paid for so skipping"); - continue; - }; - - upload_tasks.push(async move { - self_clone - .chunk_upload_with_payment(chunk, proof.clone()) - .await - .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}")) - // Return chunk reference too, to re-use it next attempt/iteration - .map_err(|err| (chunk, err)) - }); - } - let uploads = - process_tasks_with_max_concurrency(upload_tasks, *CHUNK_UPLOAD_BATCH_SIZE).await; - - // Check for errors. - let total_uploads = uploads.len(); - let uploads_failed: Vec<_> = uploads.into_iter().filter_map(|up| up.err()).collect(); - info!( - "Uploaded {} chunks out of {total_uploads}", - total_uploads - uploads_failed.len() - ); - - // All uploads succeeded. - if uploads_failed.is_empty() { - return vec![]; - } - - // Max retries reached. - if current_attempt > RETRY_ATTEMPTS { - return uploads_failed; - } - - tracing::info!( - "Retrying putting {} failed chunks (attempt {current_attempt}/3)", - uploads_failed.len() - ); - - // Re-iterate over the failed chunks - chunks = uploads_failed.into_iter().map(|(chunk, _)| chunk).collect(); - current_attempt += 1; - } - } -} diff --git a/autonomi/src/client/data_types/chunk.rs b/autonomi/src/client/data_types/chunk.rs new file mode 100644 index 0000000000..06626599a2 --- /dev/null +++ b/autonomi/src/client/data_types/chunk.rs @@ -0,0 +1,406 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + client::{ + payment::{PaymentOption, Receipt}, + quote::CostError, + utils::process_tasks_with_max_concurrency, + GetError, PutError, + }, + self_encryption::DataMapLevel, + Client, +}; +use ant_evm::{Amount, AttoTokens, ProofOfPayment}; +use ant_networking::NetworkError; +use ant_protocol::{ + storage::{try_deserialize_record, try_serialize_record, DataTypes, RecordHeader, RecordKind}, + NetworkAddress, +}; +use bytes::Bytes; +use libp2p::kad::Record; +use self_encryption::{decrypt_full_set, DataMap, EncryptedChunk}; +use serde::{Deserialize, Serialize}; +use std::{ + hash::{DefaultHasher, Hash, Hasher}, + sync::LazyLock, +}; + +pub use ant_protocol::storage::{Chunk, ChunkAddress}; + +/// Number of retries to upload chunks. +pub(crate) const RETRY_ATTEMPTS: usize = 3; + +/// Number of chunks to upload in parallel. +/// +/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. +pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk upload batch size: {}", batch_size); + batch_size +}); + +/// Number of chunks to download in parallel. +/// +/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. +pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk download batch size: {}", batch_size); + batch_size +}); + +/// Private data on the network can be accessed with this +/// Uploading this data in a chunk makes it publicly accessible from the address of that Chunk +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DataMapChunk(pub(crate) Chunk); + +impl DataMapChunk { + pub fn to_hex(&self) -> String { + hex::encode(self.0.value()) + } + + pub fn from_hex(hex: &str) -> Result { + let data = hex::decode(hex)?; + Ok(Self(Chunk::new(Bytes::from(data)))) + } + + /// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side. + pub fn address(&self) -> String { + hash_to_short_string(&self.to_hex()) + } +} + +impl From for DataMapChunk { + fn from(value: Chunk) -> Self { + Self(value) + } +} + +fn hash_to_short_string(input: &str) -> String { + let mut hasher = DefaultHasher::new(); + input.hash(&mut hasher); + let hash_value = hasher.finish(); + hash_value.to_string() +} + +impl Client { + /// Get a chunk from the network. + pub async fn chunk_get(&self, addr: &ChunkAddress) -> Result { + info!("Getting chunk: {addr:?}"); + + let key = NetworkAddress::from_chunk_address(*addr).to_record_key(); + debug!("Fetching chunk from network at: {key:?}"); + + let get_cfg = self.config.chunks.get_cfg(); + let record = self + .network + .get_record_from_network(key, &get_cfg) + .await + .inspect_err(|err| error!("Error fetching chunk: {err:?}"))?; + let header = RecordHeader::from_record(&record)?; + + if let Ok(true) = RecordHeader::is_record_of_type_chunk(&record) { + let chunk: Chunk = try_deserialize_record(&record)?; + Ok(chunk) + } else { + error!( + "Record kind mismatch: expected Chunk, got {:?}", + header.kind + ); + Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly(DataTypes::Chunk)).into()) + } + } + + /// Manually upload a chunk to the network. + /// It is recommended to use the [`Client::data_put`] method instead to upload data. + pub async fn chunk_put( + &self, + chunk: &Chunk, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, ChunkAddress), PutError> { + let address = chunk.network_address(); + + // pay for the chunk storage + let xor_name = *chunk.name(); + debug!("Paying for chunk at address: {address:?}"); + let (payment_proofs, _skipped_payments) = self + .pay_for_content_addrs( + DataTypes::Chunk, + std::iter::once((xor_name, chunk.size())), + payment_option, + ) + .await + .inspect_err(|err| error!("Error paying for chunk {address:?} :{err:?}"))?; + + // verify payment was successful + let (proof, price) = match payment_proofs.get(&xor_name) { + Some((proof, price)) => (proof, price), + None => { + info!("Chunk at address: {address:?} was already paid for"); + return Ok((AttoTokens::zero(), *chunk.address())); + } + }; + let total_cost = *price; + + let payees = proof.payees(); + let record = Record { + key: address.to_record_key(), + value: try_serialize_record( + &(proof, chunk), + RecordKind::DataWithPayment(DataTypes::Chunk), + ) + .map_err(|_| { + PutError::Serialization("Failed to serialize chunk with payment".to_string()) + })? + .to_vec(), + publisher: None, + expires: None, + }; + + let stored_on_node = try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk)) + .map_err(|e| PutError::Serialization(format!("Failed to serialize chunk: {e:?}")))? + .to_vec(); + let target_record = Record { + key: address.to_record_key(), + value: stored_on_node, + publisher: None, + expires: None, + }; + + // store the chunk on the network + debug!("Storing chunk at address: {address:?} to the network"); + let put_cfg = self.config.chunks.chunk_put_cfg(target_record, payees); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to put record - chunk {address:?} to the network: {err}") + })?; + + Ok((total_cost, *chunk.address())) + } + + /// Get the cost of a chunk. + pub async fn chunk_cost(&self, addr: &ChunkAddress) -> Result { + trace!("Getting cost for chunk of {addr:?}"); + + let xor = *addr.xorname(); + let store_quote = self + .get_store_quotes( + DataTypes::Chunk, + std::iter::once((xor, Chunk::DEFAULT_MAX_SIZE)), + ) + .await?; + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + debug!("Calculated the cost to create chunk of {addr:?} is {total_cost}"); + Ok(total_cost) + } + + /// Upload chunks and retry failed uploads up to `RETRY_ATTEMPTS` times. + pub async fn upload_chunks_with_retries<'a>( + &self, + mut chunks: Vec<&'a Chunk>, + receipt: &Receipt, + ) -> Vec<(&'a Chunk, PutError)> { + let mut current_attempt: usize = 1; + + loop { + let mut upload_tasks = vec![]; + for chunk in chunks { + let self_clone = self.clone(); + let address = *chunk.address(); + + let Some((proof, _)) = receipt.get(chunk.name()) else { + debug!("Chunk at {address:?} was already paid for so skipping"); + continue; + }; + + upload_tasks.push(async move { + self_clone + .chunk_upload_with_payment(chunk, proof.clone()) + .await + .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}")) + // Return chunk reference too, to re-use it next attempt/iteration + .map_err(|err| (chunk, err)) + }); + } + let uploads = + process_tasks_with_max_concurrency(upload_tasks, *CHUNK_UPLOAD_BATCH_SIZE).await; + + // Check for errors. + let total_uploads = uploads.len(); + let uploads_failed: Vec<_> = uploads.into_iter().filter_map(|up| up.err()).collect(); + info!( + "Uploaded {} chunks out of {total_uploads}", + total_uploads - uploads_failed.len() + ); + + // All uploads succeeded. + if uploads_failed.is_empty() { + return vec![]; + } + + // Max retries reached. + if current_attempt > RETRY_ATTEMPTS { + return uploads_failed; + } + + tracing::info!( + "Retrying putting {} failed chunks (attempt {current_attempt}/3)", + uploads_failed.len() + ); + + // Re-iterate over the failed chunks + chunks = uploads_failed.into_iter().map(|(chunk, _)| chunk).collect(); + current_attempt += 1; + } + } + + pub(crate) async fn chunk_upload_with_payment( + &self, + chunk: &Chunk, + payment: ProofOfPayment, + ) -> Result { + let storing_nodes = payment.payees(); + + if storing_nodes.is_empty() { + return Err(PutError::PayeesMissing); + } + + debug!("Storing chunk: {chunk:?} to {:?}", storing_nodes); + + let key = chunk.network_address().to_record_key(); + + let record_kind = RecordKind::DataWithPayment(DataTypes::Chunk); + let record = Record { + key: key.clone(), + value: try_serialize_record(&(payment, chunk.clone()), record_kind) + .map_err(|e| { + PutError::Serialization(format!( + "Failed to serialize chunk with payment: {e:?}" + )) + })? + .to_vec(), + publisher: None, + expires: None, + }; + + let stored_on_node = try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk)) + .map_err(|e| PutError::Serialization(format!("Failed to serialize chunk: {e:?}")))? + .to_vec(); + let target_record = Record { + key, + value: stored_on_node, + publisher: None, + expires: None, + }; + + let put_cfg = self + .config + .chunks + .chunk_put_cfg(target_record, storing_nodes.clone()); + self.network.put_record(record, &put_cfg).await?; + debug!("Successfully stored chunk: {chunk:?} to {storing_nodes:?}"); + Ok(*chunk.address()) + } + + /// Unpack a wrapped data map and fetch all bytes using self-encryption. + pub(crate) async fn fetch_from_data_map_chunk( + &self, + data_map_bytes: &Bytes, + ) -> Result { + let mut data_map_level: DataMapLevel = rmp_serde::from_slice(data_map_bytes) + .map_err(GetError::InvalidDataMap) + .inspect_err(|err| error!("Error deserializing data map: {err:?}"))?; + + loop { + let data_map = match &data_map_level { + DataMapLevel::First(map) => map, + DataMapLevel::Additional(map) => map, + }; + let data = self.fetch_from_data_map(data_map).await?; + + match &data_map_level { + DataMapLevel::First(_) => break Ok(data), + DataMapLevel::Additional(_) => { + data_map_level = rmp_serde::from_slice(&data).map_err(|err| { + error!("Error deserializing data map: {err:?}"); + GetError::InvalidDataMap(err) + })?; + continue; + } + }; + } + } + + /// Fetch and decrypt all chunks in the data map. + pub(crate) async fn fetch_from_data_map(&self, data_map: &DataMap) -> Result { + debug!("Fetching encrypted data chunks from data map {data_map:?}"); + let mut download_tasks = vec![]; + for info in data_map.infos() { + download_tasks.push(async move { + match self + .chunk_get(&ChunkAddress::new(info.dst_hash)) + .await + .inspect_err(|err| { + error!( + "Error fetching chunk {:?}: {err:?}", + ChunkAddress::new(info.dst_hash) + ) + }) { + Ok(chunk) => Ok(EncryptedChunk { + index: info.index, + content: chunk.value, + }), + Err(err) => { + error!( + "Error fetching chunk {:?}: {err:?}", + ChunkAddress::new(info.dst_hash) + ); + Err(err) + } + } + }); + } + debug!("Successfully fetched all the encrypted chunks"); + let encrypted_chunks = + process_tasks_with_max_concurrency(download_tasks, *CHUNK_DOWNLOAD_BATCH_SIZE) + .await + .into_iter() + .collect::, GetError>>()?; + + let data = decrypt_full_set(data_map, &encrypted_chunks).map_err(|e| { + error!("Error decrypting encrypted_chunks: {e:?}"); + GetError::Decryption(crate::self_encryption::Error::SelfEncryption(e)) + })?; + debug!("Successfully decrypted all the chunks"); + Ok(data) + } +} diff --git a/autonomi/src/client/data_types/graph.rs b/autonomi/src/client/data_types/graph.rs new file mode 100644 index 0000000000..f86c004022 --- /dev/null +++ b/autonomi/src/client/data_types/graph.rs @@ -0,0 +1,191 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::payment::PayError; +use crate::client::payment::PaymentOption; +use crate::client::quote::CostError; +use crate::client::Client; +use crate::client::ClientEvent; +use crate::client::UploadSummary; + +use ant_evm::{Amount, AttoTokens, EvmWalletError}; +use ant_networking::get_graph_entry_from_record; +use ant_networking::GetRecordError; +use ant_networking::NetworkError; +use ant_protocol::PrettyPrintRecordKey; +use ant_protocol::{ + storage::{try_serialize_record, DataTypes, RecordKind}, + NetworkAddress, +}; +use bls::PublicKey; +use libp2p::kad::Record; + +pub use crate::SecretKey; +pub use ant_protocol::storage::{GraphContent, GraphEntry, GraphEntryAddress}; + +#[derive(Debug, thiserror::Error)] +pub enum GraphError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("Network error")] + Network(#[from] NetworkError), + #[error("Serialization error")] + Serialization, + #[error("Verification failed (corrupt)")] + FailedVerification, + #[error("Payment failure occurred during creation.")] + Pay(#[from] PayError), + #[error("Failed to retrieve wallet payment")] + Wallet(#[from] EvmWalletError), + #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another transaction name")] + InvalidQuote, + #[error("Entry already exists at this address: {0:?}")] + AlreadyExists(GraphEntryAddress), + #[error("Graph forked! Multiple entries found: {0:?}")] + Fork(Vec), +} + +impl Client { + /// Fetches a GraphEntry from the network. + pub async fn graph_entry_get( + &self, + address: &GraphEntryAddress, + ) -> Result { + let key = NetworkAddress::from_graph_entry_address(*address).to_record_key(); + let get_cfg = self.config.graph_entry.get_cfg(); + let record = self + .network + .get_record_from_network(key.clone(), &get_cfg) + .await?; + debug!( + "Got record from the network, {:?}", + PrettyPrintRecordKey::from(&record.key) + ); + + let graph_entries = get_graph_entry_from_record(&record)?; + match &graph_entries[..] { + [entry] => Ok(entry.clone()), + multiple => Err(GraphError::Fork(multiple.to_vec())), + } + } + + /// Check if a graph_entry exists on the network + pub async fn graph_entry_check_existance( + &self, + address: &GraphEntryAddress, + ) -> Result { + let key = NetworkAddress::from_graph_entry_address(*address).to_record_key(); + debug!("Checking graph_entry existance at: {key:?}"); + let get_cfg = self.config.graph_entry.verification_cfg(); + match self + .network + .get_record_from_network(key.clone(), &get_cfg) + .await + { + Ok(_) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { .. })) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::RecordNotFound)) => Ok(false), + Err(err) => Err(GraphError::Network(err)) + .inspect_err(|err| error!("Error checking graph_entry existance: {err:?}")), + } + } + + /// Manually puts a GraphEntry to the network. + pub async fn graph_entry_put( + &self, + entry: GraphEntry, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, GraphEntryAddress), GraphError> { + let address = entry.address(); + + // pay for the graph entry + let xor_name = address.xorname(); + debug!("Paying for graph entry at address: {address:?}"); + let (payment_proofs, skipped_payments) = self + .pay_for_content_addrs( + DataTypes::GraphEntry, + std::iter::once((*xor_name, entry.size())), + payment_option, + ) + .await + .inspect_err(|err| { + error!("Failed to pay for graph entry at address: {address:?} : {err}") + })?; + + // make sure the graph entry was paid for + let (proof, price) = match payment_proofs.get(xor_name) { + Some((proof, price)) => (proof, price), + None => { + // graph entry was skipped, meaning it was already paid for + error!("GraphEntry at address: {address:?} was already paid for"); + return Err(GraphError::AlreadyExists(address)); + } + }; + let total_cost = *price; + + // prepare the record for network storage + let payees = proof.payees(); + let record = Record { + key: NetworkAddress::from_graph_entry_address(address).to_record_key(), + value: try_serialize_record( + &(proof, &entry), + RecordKind::DataWithPayment(DataTypes::GraphEntry), + ) + .map_err(|_| GraphError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + let put_cfg = self.config.graph_entry.put_cfg(Some(payees)); + + // put the record to the network + debug!("Storing GraphEntry at address {address:?} to the network"); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to put record - GraphEntry {address:?} to the network: {err}") + })?; + + // send client event + if let Some(channel) = self.client_event_sender.as_ref() { + let summary = UploadSummary { + records_paid: 1usize.saturating_sub(skipped_payments), + records_already_paid: skipped_payments, + tokens_spent: price.as_atto(), + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err}"); + } + } + + Ok((total_cost, address)) + } + + /// Get the cost to create a GraphEntry + pub async fn graph_entry_cost(&self, key: &PublicKey) -> Result { + trace!("Getting cost for GraphEntry of {key:?}"); + let address = GraphEntryAddress::from_owner(*key); + let xor = *address.xorname(); + let store_quote = self + .get_store_quotes( + DataTypes::GraphEntry, + std::iter::once((xor, GraphEntry::MAX_SIZE)), + ) + .await?; + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + debug!("Calculated the cost to create GraphEntry of {key:?} is {total_cost}"); + Ok(total_cost) + } +} diff --git a/autonomi/src/client/data_types/mod.rs b/autonomi/src/client/data_types/mod.rs new file mode 100644 index 0000000000..0f4b4d8be1 --- /dev/null +++ b/autonomi/src/client/data_types/mod.rs @@ -0,0 +1,12 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +pub mod chunk; +pub mod graph; +pub mod pointer; +pub mod scratchpad; diff --git a/autonomi/src/client/data_types/pointer.rs b/autonomi/src/client/data_types/pointer.rs new file mode 100644 index 0000000000..5ac9ada854 --- /dev/null +++ b/autonomi/src/client/data_types/pointer.rs @@ -0,0 +1,281 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::{ + payment::{PayError, PaymentOption}, + quote::CostError, + Client, +}; +use ant_evm::{Amount, AttoTokens, EvmWalletError}; +use ant_networking::{GetRecordError, NetworkError}; +use ant_protocol::{ + storage::{try_deserialize_record, try_serialize_record, DataTypes, RecordHeader, RecordKind}, + NetworkAddress, +}; +use bls::{PublicKey, SecretKey}; +use libp2p::kad::Record; +use tracing::{debug, error, trace}; + +pub use ant_protocol::storage::{Pointer, PointerAddress, PointerTarget}; + +/// Errors that can occur when dealing with Pointers +#[derive(Debug, thiserror::Error)] +pub enum PointerError { + #[error("Network error")] + Network(#[from] NetworkError), + #[error("Serialization error")] + Serialization, + #[error("Pointer record corrupt: {0}")] + Corrupt(String), + #[error("Pointer signature is invalid")] + BadSignature, + #[error("Payment failure occurred during pointer creation.")] + Pay(#[from] PayError), + #[error("Failed to retrieve wallet payment")] + Wallet(#[from] EvmWalletError), + #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another pointer name")] + InvalidQuote, + #[error("Pointer already exists at this address: {0:?}")] + PointerAlreadyExists(PointerAddress), + #[error("Pointer cannot be updated as it does not exist, please create it first or wait for it to be created")] + CannotUpdateNewPointer, +} + +impl Client { + /// Get a pointer from the network + pub async fn pointer_get(&self, address: &PointerAddress) -> Result { + let key = NetworkAddress::from_pointer_address(*address).to_record_key(); + debug!("Fetching pointer from network at: {key:?}"); + + let get_cfg = self.config.pointer.get_cfg(); + let record = self + .network + .get_record_from_network(key.clone(), &get_cfg) + .await + .inspect_err(|err| error!("Error fetching pointer: {err:?}"))?; + let header = RecordHeader::from_record(&record).map_err(|err| { + PointerError::Corrupt(format!( + "Failed to parse record header for pointer at {key:?}: {err:?}" + )) + })?; + + let kind = header.kind; + if !matches!(kind, RecordKind::DataOnly(DataTypes::Pointer)) { + error!("Record kind mismatch: expected Pointer, got {kind:?}"); + return Err( + NetworkError::RecordKindMismatch(RecordKind::DataOnly(DataTypes::Pointer)).into(), + ); + }; + + let pointer: Pointer = try_deserialize_record(&record).map_err(|err| { + PointerError::Corrupt(format!( + "Failed to parse record for pointer at {key:?}: {err:?}" + )) + })?; + + Self::pointer_verify(&pointer)?; + Ok(pointer) + } + + /// Check if a pointer exists on the network + pub async fn pointer_check_existance( + &self, + address: &PointerAddress, + ) -> Result { + let key = NetworkAddress::from_pointer_address(*address).to_record_key(); + debug!("Checking pointer existance at: {key:?}"); + let get_cfg = self.config.pointer.verification_cfg(); + match self + .network + .get_record_from_network(key.clone(), &get_cfg) + .await + { + Ok(_) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { .. })) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::RecordNotFound)) => Ok(false), + Err(err) => Err(PointerError::Network(err)) + .inspect_err(|err| error!("Error checking pointer existance: {err:?}")), + } + } + + /// Verify a pointer + pub fn pointer_verify(pointer: &Pointer) -> Result<(), PointerError> { + if !pointer.verify_signature() { + return Err(PointerError::BadSignature); + } + Ok(()) + } + + /// Manually store a pointer on the network + pub async fn pointer_put( + &self, + pointer: Pointer, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, PointerAddress), PointerError> { + let address = pointer.address(); + + // pay for the pointer storage + let xor_name = *address.xorname(); + debug!("Paying for pointer at address: {address:?}"); + let (payment_proofs, _skipped_payments) = self + .pay_for_content_addrs( + DataTypes::Pointer, + std::iter::once((xor_name, Pointer::size())), + payment_option, + ) + .await + .inspect_err(|err| { + error!("Failed to pay for pointer at address: {address:?} : {err}") + })?; + + // verify payment was successful + let (proof, price) = match payment_proofs.get(&xor_name) { + Some((proof, price)) => (Some(proof), price), + None => { + info!("Pointer at address: {address:?} was already paid for, update is free"); + (None, &AttoTokens::zero()) + } + }; + let total_cost = *price; + + let (record, payees) = if let Some(proof) = proof { + let payees = Some(proof.payees()); + let record = Record { + key: NetworkAddress::from_pointer_address(address).to_record_key(), + value: try_serialize_record( + &(proof, &pointer), + RecordKind::DataWithPayment(DataTypes::Pointer), + ) + .map_err(|_| PointerError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + (record, payees) + } else { + let record = Record { + key: NetworkAddress::from_pointer_address(address).to_record_key(), + value: try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer)) + .map_err(|_| PointerError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + (record, None) + }; + + // store the pointer on the network + debug!("Storing pointer at address {address:?} to the network"); + let put_cfg = self.config.pointer.put_cfg(payees); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to put record - pointer {address:?} to the network: {err}") + })?; + + Ok((total_cost, address)) + } + + /// Create a new pointer on the network. + /// + /// Make sure that the owner key is not already used for another pointer as each key is associated with one pointer + pub async fn pointer_create( + &self, + owner: &SecretKey, + target: PointerTarget, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, PointerAddress), PointerError> { + let address = PointerAddress::from_owner(owner.public_key()); + let already_exists = self.pointer_check_existance(&address).await?; + if already_exists { + return Err(PointerError::PointerAlreadyExists(address)); + } + + let pointer = Pointer::new(owner, 0, target); + self.pointer_put(pointer, payment_option).await + } + + /// Update an existing pointer to point to a new target on the network. + /// + /// The pointer needs to be created first with [`Client::pointer_put`]. + /// This operation is free as the pointer was already paid for at creation. + /// Only the latest version of the pointer is kept on the Network, previous versions will be overwritten and unrecoverable. + pub async fn pointer_update( + &self, + owner: &SecretKey, + target: PointerTarget, + ) -> Result<(), PointerError> { + let address = PointerAddress::from_owner(owner.public_key()); + let current = match self.pointer_get(&address).await { + Ok(pointer) => Some(pointer), + Err(PointerError::Network(NetworkError::GetRecordError( + GetRecordError::RecordNotFound, + ))) => None, + Err(PointerError::Network(NetworkError::GetRecordError( + GetRecordError::SplitRecord { result_map }, + ))) => result_map + .values() + .filter_map(|(record, _)| try_deserialize_record::(record).ok()) + .max_by_key(|pointer: &Pointer| pointer.counter()), + Err(err) => { + return Err(err); + } + }; + + let pointer = if let Some(p) = current { + let version = p.counter() + 1; + Pointer::new(owner, version, target) + } else { + warn!("Pointer at address {address:?} cannot be updated as it does not exist, please create it first or wait for it to be created"); + return Err(PointerError::CannotUpdateNewPointer); + }; + + // prepare the record to be stored + let record = Record { + key: NetworkAddress::from_pointer_address(address).to_record_key(), + value: try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer)) + .map_err(|_| PointerError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + + // store the pointer on the network + debug!("Updating pointer at address {address:?} to the network"); + let put_cfg = self.config.pointer.put_cfg(None); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to update pointer at address {address:?} to the network: {err}") + })?; + + Ok(()) + } + + /// Calculate the cost of storing a pointer + pub async fn pointer_cost(&self, key: &PublicKey) -> Result { + trace!("Getting cost for pointer of {key:?}"); + + let address = PointerAddress::from_owner(*key); + let xor = *address.xorname(); + let store_quote = self + .get_store_quotes(DataTypes::Pointer, std::iter::once((xor, Pointer::size()))) + .await?; + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + debug!("Calculated the cost to create pointer of {key:?} is {total_cost}"); + Ok(total_cost) + } +} diff --git a/autonomi/src/client/data_types/scratchpad.rs b/autonomi/src/client/data_types/scratchpad.rs new file mode 100644 index 0000000000..35c63dc841 --- /dev/null +++ b/autonomi/src/client/data_types/scratchpad.rs @@ -0,0 +1,338 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::payment::{PayError, PaymentOption}; +use crate::{client::quote::CostError, Client}; +use crate::{Amount, AttoTokens}; +use ant_networking::{GetRecordError, NetworkError}; +use ant_protocol::storage::{try_serialize_record, RecordKind}; +use ant_protocol::{ + storage::{try_deserialize_record, DataTypes}, + NetworkAddress, +}; +use libp2p::kad::Record; + +pub use crate::Bytes; +pub use ant_protocol::storage::{Scratchpad, ScratchpadAddress}; +pub use bls::{PublicKey, SecretKey, Signature}; + +const SCRATCHPAD_MAX_SIZE: usize = Scratchpad::MAX_SIZE; + +/// Errors that can occur when dealing with Scratchpads +#[derive(Debug, thiserror::Error)] +pub enum ScratchpadError { + #[error("Payment failure occurred during scratchpad creation.")] + Pay(#[from] PayError), + #[error("Scratchpad found at {0:?} was not a valid record.")] + CouldNotDeserializeScratchPad(ScratchpadAddress), + #[error("Network: {0}")] + Network(#[from] NetworkError), + #[error("Scratchpad not found")] + Missing, + #[error("Serialization error")] + Serialization, + #[error("Scratchpad already exists at this address: {0:?}")] + ScratchpadAlreadyExists(ScratchpadAddress), + #[error("Scratchpad cannot be updated as it does not exist, please create it first or wait for it to be created")] + CannotUpdateNewScratchpad, + #[error("Scratchpad size is too big: {0} > {SCRATCHPAD_MAX_SIZE}")] + ScratchpadTooBig(usize), + #[error("Scratchpad signature is not valid")] + BadSignature, +} + +impl Client { + /// Get Scratchpad from the Network. + /// A Scratchpad is stored at the owner's public key so we can derive the address from it. + pub async fn scratchpad_get_from_public_key( + &self, + public_key: &PublicKey, + ) -> Result { + let address = ScratchpadAddress::new(*public_key); + self.scratchpad_get(&address).await + } + + /// Get Scratchpad from the Network + pub async fn scratchpad_get( + &self, + address: &ScratchpadAddress, + ) -> Result { + let network_address = NetworkAddress::from_scratchpad_address(*address); + info!("Fetching scratchpad from network at {network_address:?}",); + let scratch_key = network_address.to_record_key(); + let get_cfg = self.config.scratchpad.get_cfg(); + let pad = match self + .network + .get_record_from_network(scratch_key.clone(), &get_cfg) + .await + { + Ok(record) => { + debug!("Got scratchpad for {scratch_key:?}"); + try_deserialize_record::(&record) + .map_err(|_| ScratchpadError::CouldNotDeserializeScratchPad(*address))? + } + Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { result_map })) => { + debug!("Got multiple scratchpads for {scratch_key:?}"); + let mut pads = result_map + .values() + .map(|(record, _)| try_deserialize_record::(record)) + .collect::, _>>() + .map_err(|_| ScratchpadError::CouldNotDeserializeScratchPad(*address))?; + + // take the latest versions + pads.sort_by_key(|s| s.counter()); + let max_version = pads.last().map(|p| p.counter()).unwrap_or_else(|| { + error!("Got empty scratchpad vector for {scratch_key:?}"); + u64::MAX + }); + let latest_pads: Vec<_> = pads + .into_iter() + .filter(|s| s.counter() == max_version) + .collect(); + + // make sure we only have one of latest version + let pad = match &latest_pads[..] { + [one] => one, + [multi, ..] => { + error!("Got multiple conflicting scratchpads for {scratch_key:?} with the latest version, returning the first one"); + multi + } + [] => { + error!("Got empty scratchpad vector for {scratch_key:?}"); + return Err(ScratchpadError::Missing); + } + }; + pad.to_owned() + } + Err(e) => { + warn!("Failed to fetch scratchpad {network_address:?} from network: {e}"); + return Err(e)?; + } + }; + + Self::scratchpad_verify(&pad)?; + Ok(pad) + } + + /// Check if a scratchpad exists on the network + pub async fn scratchpad_check_existance( + &self, + address: &ScratchpadAddress, + ) -> Result { + let key = NetworkAddress::from_scratchpad_address(*address).to_record_key(); + debug!("Checking scratchpad existance at: {key:?}"); + let get_cfg = self.config.scratchpad.verification_cfg(); + match self + .network + .get_record_from_network(key.clone(), &get_cfg) + .await + { + Ok(_) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { .. })) => Ok(true), + Err(NetworkError::GetRecordError(GetRecordError::RecordNotFound)) => Ok(false), + Err(err) => Err(ScratchpadError::Network(err)) + .inspect_err(|err| error!("Error checking scratchpad existance: {err:?}")), + } + } + + /// Verify a scratchpad + pub fn scratchpad_verify(scratchpad: &Scratchpad) -> Result<(), ScratchpadError> { + if !scratchpad.verify_signature() { + return Err(ScratchpadError::BadSignature); + } + if scratchpad.is_too_big() { + return Err(ScratchpadError::ScratchpadTooBig(scratchpad.size())); + } + Ok(()) + } + + /// Manually store a scratchpad on the network + pub async fn scratchpad_put( + &self, + scratchpad: Scratchpad, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, ScratchpadAddress), ScratchpadError> { + let address = scratchpad.address(); + Self::scratchpad_verify(&scratchpad)?; + + // pay for the scratchpad + let xor_name = address.xorname(); + debug!("Paying for scratchpad at address: {address:?}"); + let (payment_proofs, _skipped_payments) = self + .pay_for_content_addrs( + DataTypes::Scratchpad, + std::iter::once((xor_name, scratchpad.size())), + payment_option, + ) + .await + .inspect_err(|err| { + error!("Failed to pay for scratchpad at address: {address:?} : {err}") + })?; + + // verify payment was successful + let (proof, price) = match payment_proofs.get(&xor_name) { + Some((proof, price)) => (Some(proof), price), + None => { + info!("Scratchpad at address: {address:?} was already paid for, update is free"); + (None, &AttoTokens::zero()) + } + }; + let total_cost = *price; + + let net_addr = NetworkAddress::from_scratchpad_address(*address); + let (record, payees) = if let Some(proof) = proof { + let payees = Some(proof.payees()); + let record = Record { + key: net_addr.to_record_key(), + value: try_serialize_record( + &(proof, &scratchpad), + RecordKind::DataWithPayment(DataTypes::Scratchpad), + ) + .map_err(|_| ScratchpadError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + (record, payees) + } else { + let record = Record { + key: net_addr.to_record_key(), + value: try_serialize_record( + &scratchpad, + RecordKind::DataOnly(DataTypes::Scratchpad), + ) + .map_err(|_| ScratchpadError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + (record, None) + }; + + // store the scratchpad on the network + debug!("Storing scratchpad at address {address:?} to the network"); + let put_cfg = self.config.scratchpad.put_cfg(payees); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to put record - scratchpad {address:?} to the network: {err}") + })?; + + Ok((total_cost, *address)) + } + + /// Create a new scratchpad to the network. + /// + /// Make sure that the owner key is not already used for another scratchpad as each key is associated with one scratchpad. + /// The data will be encrypted with the owner key before being stored on the network. + /// The content type is used to identify the type of data stored in the scratchpad, the choice is up to the caller. + /// + /// Returns the cost and the address of the scratchpad. + pub async fn scratchpad_create( + &self, + owner: &SecretKey, + content_type: u64, + initial_data: &Bytes, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, ScratchpadAddress), ScratchpadError> { + let address = ScratchpadAddress::new(owner.public_key()); + let already_exists = self.scratchpad_check_existance(&address).await?; + if already_exists { + return Err(ScratchpadError::ScratchpadAlreadyExists(address)); + } + + let counter = 0; + let scratchpad = Scratchpad::new(owner, content_type, initial_data, counter); + self.scratchpad_put(scratchpad, payment_option).await + } + + /// Update an existing scratchpad to the network. + /// The scratchpad needs to be created first with [`Client::scratchpad_create`]. + /// This operation is free as the scratchpad was already paid for at creation. + /// Only the latest version of the scratchpad is kept on the Network, previous versions will be overwritten and unrecoverable. + pub async fn scratchpad_update( + &self, + owner: &SecretKey, + content_type: u64, + data: &Bytes, + ) -> Result<(), ScratchpadError> { + let address = ScratchpadAddress::new(owner.public_key()); + let current = match self.scratchpad_get(&address).await { + Ok(scratchpad) => Some(scratchpad), + Err(ScratchpadError::Network(NetworkError::GetRecordError( + GetRecordError::RecordNotFound, + ))) => None, + Err(ScratchpadError::Network(NetworkError::GetRecordError( + GetRecordError::SplitRecord { result_map }, + ))) => result_map + .values() + .filter_map(|(record, _)| try_deserialize_record::(record).ok()) + .max_by_key(|scratchpad: &Scratchpad| scratchpad.counter()), + Err(err) => { + return Err(err); + } + }; + + let scratchpad = if let Some(p) = current { + let version = p.counter() + 1; + Scratchpad::new(owner, content_type, data, version) + } else { + warn!("Scratchpad at address {address:?} cannot be updated as it does not exist, please create it first or wait for it to be created"); + return Err(ScratchpadError::CannotUpdateNewScratchpad); + }; + + // make sure the scratchpad is valid + Self::scratchpad_verify(&scratchpad)?; + + // prepare the record to be stored + let record = Record { + key: NetworkAddress::from_scratchpad_address(address).to_record_key(), + value: try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad)) + .map_err(|_| ScratchpadError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + + // store the scratchpad on the network + let put_cfg = self.config.scratchpad.put_cfg(None); + debug!("Updating scratchpad at address {address:?} to the network"); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to update scratchpad at address {address:?} to the network: {err}") + })?; + + Ok(()) + } + + /// Get the cost of creating a new Scratchpad + pub async fn scratchpad_cost(&self, owner: &PublicKey) -> Result { + info!("Getting cost for scratchpad"); + let scratch_xor = ScratchpadAddress::new(*owner).xorname(); + + let store_quote = self + .get_store_quotes( + DataTypes::Scratchpad, + std::iter::once((scratch_xor, SCRATCHPAD_MAX_SIZE)), + ) + .await?; + + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + + Ok(total_cost) + } +} diff --git a/autonomi/src/client/external_signer.rs b/autonomi/src/client/external_signer.rs index 30114712f3..24c23ae35a 100644 --- a/autonomi/src/client/external_signer.rs +++ b/autonomi/src/client/external_signer.rs @@ -1,4 +1,5 @@ -use crate::client::data::PutError; +use crate::client::quote::DataTypes; +use crate::client::PutError; use crate::self_encryption::encrypt; use crate::Client; use ant_evm::QuotePayment; @@ -17,7 +18,8 @@ impl Client { /// Returns a cost map, data payments to be executed and a list of free (already paid for) chunks. pub async fn get_quotes_for_content_addresses( &self, - content_addrs: impl Iterator + Clone, + data_type: DataTypes, + content_addrs: impl Iterator + Clone, ) -> Result< ( HashMap, @@ -26,14 +28,20 @@ impl Client { ), PutError, > { - let quote = self.get_store_quotes(content_addrs.clone()).await?; + let quote = self + .get_store_quotes(data_type, content_addrs.clone()) + .await?; let payments = quote.payments(); - let free_chunks = content_addrs - .filter(|addr| !quote.0.contains_key(addr)) + let free_chunks: Vec<_> = content_addrs + .filter(|(addr, _)| !quote.0.contains_key(addr)) .collect(); let quotes_per_addr: HashMap<_, _> = quote.0.into_iter().collect(); - Ok((quotes_per_addr, payments, free_chunks)) + Ok(( + quotes_per_addr, + payments, + free_chunks.iter().map(|(addr, _)| *addr).collect(), + )) } } @@ -41,7 +49,7 @@ impl Client { /// /// Returns the data map chunk and file chunks. pub fn encrypt_data(data: Bytes) -> Result<(Chunk, Vec), PutError> { - let now = ant_networking::target_arch::Instant::now(); + let now = ant_networking::time::Instant::now(); let result = encrypt(data)?; debug!("Encryption took: {:.2?}", now.elapsed()); diff --git a/autonomi/src/client/files/fs.rs b/autonomi/src/client/files/fs.rs deleted file mode 100644 index 2428f2d344..0000000000 --- a/autonomi/src/client/files/fs.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use super::archive::{PrivateArchive, PrivateArchiveAccess}; -use crate::client::data::{CostError, DataMapChunk, GetError, PutError}; -use crate::client::files::get_relative_file_path_from_abs_file_and_folder_path; -use crate::client::utils::process_tasks_with_max_concurrency; -use crate::client::Client; -use ant_evm::EvmWallet; -use bytes::Bytes; -use std::{path::PathBuf, sync::LazyLock}; - -/// Number of files to upload in parallel. -/// -/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. -pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("File upload batch size: {}", batch_size); - batch_size -}); - -/// Errors that can occur during the file upload operation. -#[derive(Debug, thiserror::Error)] -pub enum UploadError { - #[error("Failed to recursively traverse directory")] - WalkDir(#[from] walkdir::Error), - #[error("Input/output failure")] - IoError(#[from] std::io::Error), - #[error("Failed to upload file")] - PutError(#[from] PutError), - #[error("Failed to fetch file")] - GetError(#[from] GetError), - #[error("Failed to serialize")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), -} - -/// Errors that can occur during the download operation. -#[derive(Debug, thiserror::Error)] -pub enum DownloadError { - #[error("Failed to download file")] - GetError(#[from] GetError), - #[error("IO failure")] - IoError(#[from] std::io::Error), -} - -/// Errors that can occur during the file cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum FileCostError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("IO failure")] - IoError(#[from] std::io::Error), - #[error("Serialization error")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Self encryption error")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Walkdir error")] - WalkDir(#[from] walkdir::Error), -} - -impl Client { - /// Download a private file from network to local file system - pub async fn file_download( - &self, - data_access: DataMapChunk, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let data = self.data_get(data_access).await?; - if let Some(parent) = to_dest.parent() { - tokio::fs::create_dir_all(parent).await?; - debug!("Created parent directories for {to_dest:?}"); - } - tokio::fs::write(to_dest.clone(), data).await?; - debug!("Downloaded file to {to_dest:?}"); - Ok(()) - } - - /// Download a private directory from network to local file system - pub async fn dir_download( - &self, - archive_access: PrivateArchiveAccess, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let archive = self.archive_get(archive_access).await?; - for (path, addr, _meta) in archive.iter() { - self.file_download(addr.clone(), to_dest.join(path)).await?; - } - debug!("Downloaded directory to {to_dest:?}"); - Ok(()) - } - - /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. - /// The data maps of these (private) files are not uploaded but returned within the [`PrivateArchive`] return type. - pub async fn dir_upload( - &self, - dir_path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading directory as private: {dir_path:?}"); - let start = tokio::time::Instant::now(); - - // start upload of file in parallel - let mut upload_tasks = Vec::new(); - for entry in walkdir::WalkDir::new(dir_path.clone()) { - let entry = entry?; - if !entry.file_type().is_file() { - continue; - } - - let metadata = super::fs_public::metadata_from_entry(&entry); - let path = entry.path().to_path_buf(); - upload_tasks.push(async move { - let file = self.file_upload(path.clone(), wallet).await; - (path, metadata, file) - }); - } - - // wait for all files to be uploaded - let uploads = - process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; - info!( - "Upload of {} files completed in {:?}", - uploads.len(), - start.elapsed() - ); - let mut archive = PrivateArchive::new(); - for (path, metadata, maybe_file) in uploads.into_iter() { - let rel_path = get_relative_file_path_from_abs_file_and_folder_path(&path, &dir_path); - - match maybe_file { - Ok(file) => archive.add_file(rel_path, file, metadata), - Err(err) => { - error!("Failed to upload file: {path:?}: {err:?}"); - return Err(err); - } - } - } - - #[cfg(feature = "loud")] - println!("Upload completed in {:?}", start.elapsed()); - Ok(archive) - } - - /// Same as [`Client::dir_upload`] but also uploads the archive (privately) to the network. - /// - /// Returns the [`PrivateArchiveAccess`] allowing the private archive to be downloaded from the network. - pub async fn dir_and_archive_upload( - &self, - dir_path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - let archive = self.dir_upload(dir_path, wallet).await?; - let archive_addr = self.archive_put(&archive, wallet.into()).await?; - Ok(archive_addr) - } - - /// Upload a private file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap) - async fn file_upload( - &self, - path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading file: {path:?}"); - #[cfg(feature = "loud")] - println!("Uploading file: {path:?}"); - - let data = tokio::fs::read(path).await?; - let data = Bytes::from(data); - let addr = self.data_put(data, wallet.into()).await?; - debug!("Uploaded file successfully in the privateAchive: {addr:?}"); - Ok(addr) - } -} diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs deleted file mode 100644 index 081445019f..0000000000 --- a/autonomi/src/client/files/fs_public.rs +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use super::archive_public::{ArchiveAddr, PublicArchive}; -use super::fs::*; -use crate::client::data::DataAddr; -use crate::client::files::archive::Metadata; -use crate::client::files::get_relative_file_path_from_abs_file_and_folder_path; -use crate::client::utils::process_tasks_with_max_concurrency; -use crate::client::Client; -use ant_evm::EvmWallet; -use ant_networking::target_arch::{Duration, SystemTime}; -use bytes::Bytes; -use std::path::PathBuf; - -impl Client { - /// Download file from network to local file system - pub async fn file_download_public( - &self, - data_addr: DataAddr, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let data = self.data_get_public(data_addr).await?; - if let Some(parent) = to_dest.parent() { - tokio::fs::create_dir_all(parent).await?; - debug!("Created parent directories {parent:?} for {to_dest:?}"); - } - tokio::fs::write(to_dest.clone(), data).await?; - debug!("Downloaded file to {to_dest:?} from the network address {data_addr:?}"); - Ok(()) - } - - /// Download directory from network to local file system - pub async fn dir_download_public( - &self, - archive_addr: ArchiveAddr, - to_dest: PathBuf, - ) -> Result<(), DownloadError> { - let archive = self.archive_get_public(archive_addr).await?; - debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); - for (path, addr, _meta) in archive.iter() { - self.file_download_public(*addr, to_dest.join(path)).await?; - } - debug!( - "All files in the directory downloaded to {:?} from the network address {:?}", - to_dest.parent(), - archive_addr - ); - Ok(()) - } - - /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. - /// - /// The data maps of these files are uploaded on the network, making the individual files publicly available. - /// - /// This returns, but does not upload (!),the [`PublicArchive`] containing the data maps of the uploaded files. - pub async fn dir_upload_public( - &self, - dir_path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading directory: {dir_path:?}"); - let start = tokio::time::Instant::now(); - - // start upload of files in parallel - let mut upload_tasks = Vec::new(); - for entry in walkdir::WalkDir::new(dir_path.clone()) { - let entry = entry?; - if !entry.file_type().is_file() { - continue; - } - - let metadata = metadata_from_entry(&entry); - let path = entry.path().to_path_buf(); - upload_tasks.push(async move { - let file = self.file_upload_public(path.clone(), wallet).await; - (path, metadata, file) - }); - } - - // wait for all files to be uploaded - let uploads = - process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; - info!( - "Upload of {} files completed in {:?}", - uploads.len(), - start.elapsed() - ); - let mut archive = PublicArchive::new(); - for (path, metadata, maybe_file) in uploads.into_iter() { - let rel_path = get_relative_file_path_from_abs_file_and_folder_path(&path, &dir_path); - - match maybe_file { - Ok(file) => archive.add_file(rel_path, file, metadata), - Err(err) => { - error!("Failed to upload file: {path:?}: {err:?}"); - return Err(err); - } - } - } - - #[cfg(feature = "loud")] - println!("Upload completed in {:?}", start.elapsed()); - Ok(archive) - } - - /// Same as [`Client::dir_upload_public`] but also uploads the archive to the network. - /// - /// Returns the [`ArchiveAddr`] of the uploaded archive. - pub async fn dir_and_archive_upload_public( - &self, - dir_path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - let archive = self.dir_upload_public(dir_path, wallet).await?; - let archive_addr = self.archive_put_public(&archive, wallet).await?; - Ok(archive_addr) - } - - /// Upload a file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) - async fn file_upload_public( - &self, - path: PathBuf, - wallet: &EvmWallet, - ) -> Result { - info!("Uploading file: {path:?}"); - #[cfg(feature = "loud")] - println!("Uploading file: {path:?}"); - - let data = tokio::fs::read(path.clone()).await?; - let data = Bytes::from(data); - let addr = self.data_put_public(data, wallet.into()).await?; - debug!("File {path:?} uploaded to the network at {addr:?}"); - Ok(addr) - } - - /// Get the cost to upload a file/dir to the network. - /// quick and dirty implementation, please refactor once files are cleanly implemented - pub async fn file_cost(&self, path: &PathBuf) -> Result { - let mut archive = PublicArchive::new(); - let mut total_cost = ant_evm::Amount::ZERO; - - for entry in walkdir::WalkDir::new(path) { - let entry = entry?; - - if !entry.file_type().is_file() { - continue; - } - - let path = entry.path().to_path_buf(); - tracing::info!("Cost for file: {path:?}"); - - let data = tokio::fs::read(&path).await?; - let file_bytes = Bytes::from(data); - let file_cost = self.data_cost(file_bytes.clone()).await?; - - total_cost += file_cost.as_atto(); - - // re-do encryption to get the correct map xorname here - // this code needs refactor - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, _) = crate::self_encryption::encrypt(file_bytes)?; - tracing::debug!("Encryption took: {:.2?}", now.elapsed()); - let map_xor_name = *data_map_chunk.address().xorname(); - - let metadata = metadata_from_entry(&entry); - archive.add_file(path, map_xor_name, metadata); - } - - let root_serialized = rmp_serde::to_vec(&archive)?; - - let archive_cost = self.data_cost(Bytes::from(root_serialized)).await?; - - total_cost += archive_cost.as_atto(); - debug!("Total cost for the directory: {total_cost:?}"); - Ok(total_cost.into()) - } -} - -// Get metadata from directory entry. Defaults to `0` for creation and modification times if -// any error is encountered. Logs errors upon error. -pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { - let fs_metadata = match entry.metadata() { - Ok(metadata) => metadata, - Err(err) => { - tracing::warn!( - "Failed to get metadata for `{}`: {err}", - entry.path().display() - ); - return Metadata { - created: 0, - modified: 0, - size: 0, - }; - } - }; - - let unix_time = |property: &'static str, time: std::io::Result| { - time.inspect_err(|err| { - tracing::warn!( - "Failed to get '{property}' metadata for `{}`: {err}", - entry.path().display() - ); - }) - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH) - .inspect_err(|err| { - tracing::warn!( - "'{property}' metadata of `{}` is before UNIX epoch: {err}", - entry.path().display() - ); - }) - .unwrap_or(Duration::from_secs(0)) - .as_secs() - }; - let created = unix_time("created", fs_metadata.created()); - let modified = unix_time("modified", fs_metadata.modified()); - - Metadata { - created, - modified, - size: fs_metadata.len(), - } -} diff --git a/autonomi/src/client/files/mod.rs b/autonomi/src/client/files/mod.rs deleted file mode 100644 index a419ecfa04..0000000000 --- a/autonomi/src/client/files/mod.rs +++ /dev/null @@ -1,40 +0,0 @@ -#[cfg(feature = "fs")] -use std::path::{Path, PathBuf}; - -pub mod archive; -pub mod archive_public; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs_public; - -#[cfg(feature = "fs")] -pub(crate) fn get_relative_file_path_from_abs_file_and_folder_path( - abs_file_pah: &Path, - abs_folder_path: &Path, -) -> PathBuf { - // check if the dir is a file - let is_file = abs_folder_path.is_file(); - - // could also be the file name - let dir_name = PathBuf::from( - abs_folder_path - .file_name() - .expect("Failed to get file/dir name"), - ); - - if is_file { - dir_name - } else { - let folder_prefix = abs_folder_path - .parent() - .unwrap_or(Path::new("")) - .to_path_buf(); - abs_file_pah - .strip_prefix(folder_prefix) - .expect("Could not strip prefix path") - .to_path_buf() - } -} diff --git a/autonomi/src/client/high_level/data/mod.rs b/autonomi/src/client/high_level/data/mod.rs new file mode 100644 index 0000000000..1e7de350fc --- /dev/null +++ b/autonomi/src/client/high_level/data/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use xor_name::XorName; + +/// Private data on the network, readable only if you have the DataMapChunk +pub mod private; +/// Public data on the network, readable by anyone with the DataAddr +pub mod public; + +/// Raw Data Address (points to a DataMap) +pub type DataAddr = XorName; diff --git a/autonomi/src/client/high_level/data/private.rs b/autonomi/src/client/high_level/data/private.rs new file mode 100644 index 0000000000..f1bfaa3377 --- /dev/null +++ b/autonomi/src/client/high_level/data/private.rs @@ -0,0 +1,139 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_protocol::storage::DataTypes; + +use crate::client::payment::PaymentOption; +use crate::client::{ClientEvent, GetError, PutError, UploadSummary}; +use crate::Amount; +use crate::AttoTokens; +use crate::{self_encryption::encrypt, Client}; + +pub use crate::client::data_types::chunk::DataMapChunk; +pub use crate::Bytes; + +impl Client { + /// Fetch a blob of (private) data from the network + /// + /// # Example + /// + /// ```no_run + /// use autonomi::{Client, Bytes}; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let client = Client::init().await?; + /// # let data_map = todo!(); + /// let data_fetched = client.data_get(&data_map).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn data_get(&self, data_map: &DataMapChunk) -> Result { + info!( + "Fetching private data from Data Map {:?}", + data_map.0.address() + ); + let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; + + debug!("Successfully fetched a blob of private data from the network"); + Ok(data) + } + + /// Upload a piece of private data to the network. This data will be self-encrypted. + /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. + /// + /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. + /// + /// # Example + /// + /// ```no_run + /// use autonomi::{Client, Bytes}; + /// # #[tokio::main] + /// # async fn main() -> Result<(), Box> { + /// # let client = Client::init().await?; + /// # let wallet = todo!(); + /// let data = Bytes::from("Hello, World"); + /// let (total_cost, data_map) = client.data_put(data, wallet).await?; + /// let data_fetched = client.data_get(&data_map).await?; + /// assert_eq!(data, data_fetched); + /// # Ok(()) + /// # } + /// ``` + pub async fn data_put( + &self, + data: Bytes, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, DataMapChunk), PutError> { + let now = ant_networking::time::Instant::now(); + let (data_map_chunk, chunks) = encrypt(data)?; + debug!("Encryption took: {:.2?}", now.elapsed()); + + // Pay for all chunks + let xor_names: Vec<_> = chunks + .iter() + .map(|chunk| (*chunk.name(), chunk.size())) + .collect(); + info!("Paying for {} addresses", xor_names.len()); + let (receipt, skipped_payments) = self + .pay_for_content_addrs(DataTypes::Chunk, xor_names.into_iter(), payment_option) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}"))?; + + // Upload the chunks with the payments + debug!("Uploading {} chunks", chunks.len()); + + let mut failed_uploads = self + .upload_chunks_with_retries(chunks.iter().collect(), &receipt) + .await; + + // Return the last chunk upload error + if let Some(last_chunk_fail) = failed_uploads.pop() { + tracing::error!( + "Error uploading chunk ({:?}): {:?}", + last_chunk_fail.0.address(), + last_chunk_fail.1 + ); + return Err(last_chunk_fail.1); + } + + let record_count = chunks.len().saturating_sub(skipped_payments); + + let tokens_spent = receipt + .values() + .map(|(_, cost)| cost.as_atto()) + .sum::(); + let total_cost = AttoTokens::from_atto(tokens_spent); + + // Reporting + if let Some(channel) = self.client_event_sender.as_ref() { + let summary = UploadSummary { + records_paid: record_count, + records_already_paid: skipped_payments, + tokens_spent, + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err:?}"); + } + } + + Ok((total_cost, DataMapChunk(data_map_chunk))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::data_types::chunk::Chunk; + + #[test] + fn test_hex() { + let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello"))); + let hex = data_map.to_hex(); + let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex"); + assert_eq!(data_map, data_map2); + } +} diff --git a/autonomi/src/client/high_level/data/public.rs b/autonomi/src/client/high_level/data/public.rs new file mode 100644 index 0000000000..20fc8ed114 --- /dev/null +++ b/autonomi/src/client/high_level/data/public.rs @@ -0,0 +1,141 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_protocol::storage::DataTypes; +use bytes::Bytes; + +use crate::client::payment::PaymentOption; +use crate::client::quote::CostError; +use crate::client::{ClientEvent, GetError, PutError, UploadSummary}; +use crate::{chunk::ChunkAddress, self_encryption::encrypt, Client}; +use ant_evm::{Amount, AttoTokens}; + +use super::DataAddr; + +impl Client { + /// Fetch a blob of data from the network + pub async fn data_get_public(&self, addr: &DataAddr) -> Result { + info!("Fetching data from Data Address: {addr:?}"); + let data_map_chunk = self.chunk_get(&ChunkAddress::new(*addr)).await?; + let data = self + .fetch_from_data_map_chunk(data_map_chunk.value()) + .await?; + + debug!("Successfully fetched a blob of data from the network"); + Ok(data) + } + + /// Upload a piece of data to the network. This data is publicly accessible. + /// + /// Returns the Data Address at which the data was stored. + pub async fn data_put_public( + &self, + data: Bytes, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, DataAddr), PutError> { + let now = ant_networking::time::Instant::now(); + let (data_map_chunk, chunks) = encrypt(data)?; + let data_map_addr = data_map_chunk.address(); + debug!("Encryption took: {:.2?}", now.elapsed()); + info!("Uploading datamap chunk to the network at: {data_map_addr:?}"); + + let map_xor_name = *data_map_chunk.address().xorname(); + let mut xor_names = vec![(map_xor_name, data_map_chunk.size())]; + + for chunk in &chunks { + xor_names.push((*chunk.name(), chunk.size())); + } + + // Pay for all chunks + data map chunk + info!("Paying for {} addresses", xor_names.len()); + let (receipt, skipped_payments) = self + .pay_for_content_addrs(DataTypes::Chunk, xor_names.into_iter(), payment_option) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}"))?; + + // Upload all the chunks in parallel including the data map chunk + debug!("Uploading {} chunks", chunks.len()); + + let mut failed_uploads = self + .upload_chunks_with_retries( + chunks + .iter() + .chain(std::iter::once(&data_map_chunk)) + .collect(), + &receipt, + ) + .await; + + // Return the last chunk upload error + if let Some(last_chunk_fail) = failed_uploads.pop() { + tracing::error!( + "Error uploading chunk ({:?}): {:?}", + last_chunk_fail.0.address(), + last_chunk_fail.1 + ); + return Err(last_chunk_fail.1); + } + + let record_count = (chunks.len() + 1) - skipped_payments; + + let tokens_spent = receipt + .values() + .map(|(_proof, price)| price.as_atto()) + .sum::(); + let total_cost = AttoTokens::from_atto(tokens_spent); + + // Reporting + if let Some(channel) = self.client_event_sender.as_ref() { + let summary = UploadSummary { + records_paid: record_count, + records_already_paid: skipped_payments, + tokens_spent, + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err:?}"); + } + } + + Ok((total_cost, map_xor_name)) + } + + /// Get the estimated cost of storing a piece of data. + pub async fn data_cost(&self, data: Bytes) -> Result { + let now = ant_networking::time::Instant::now(); + let (data_map_chunks, chunks) = encrypt(data)?; + + debug!("Encryption took: {:.2?}", now.elapsed()); + + let map_xor_name = *data_map_chunks.address().xorname(); + let mut content_addrs = vec![(map_xor_name, data_map_chunks.size())]; + + for chunk in &chunks { + content_addrs.push((*chunk.name(), chunk.size())); + } + + info!( + "Calculating cost of storing {} chunks. Data map chunk at: {map_xor_name:?}", + content_addrs.len() + ); + + let store_quote = self + .get_store_quotes(DataTypes::Chunk, content_addrs.into_iter()) + .await + .inspect_err(|err| error!("Error getting store quotes: {err:?}"))?; + + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + + Ok(total_cost) + } +} diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/high_level/files/archive_private.rs similarity index 73% rename from autonomi/src/client/files/archive.rs rename to autonomi/src/client/high_level/files/archive_private.rs index 7afecec092..45a2872c69 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/high_level/files/archive_private.rs @@ -11,55 +11,24 @@ use std::{ path::{Path, PathBuf}, }; -use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; +use ant_evm::AttoTokens; +use ant_networking::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::{ client::{ - data::{DataMapChunk, GetError, PutError}, - payment::PaymentOption, + data_types::chunk::DataMapChunk, high_level::files::RenameError, payment::PaymentOption, + GetError, PutError, }, Client, }; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use thiserror::Error; + +use super::Metadata; /// Private archive data map, allowing access to the [`PrivateArchive`] data. pub type PrivateArchiveAccess = DataMapChunk; -#[derive(Error, Debug, PartialEq, Eq)] -pub enum RenameError { - #[error("File not found in archive: {0}")] - FileNotFound(PathBuf), -} - -/// Metadata for a file in an archive. Time values are UNIX timestamps. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Metadata { - /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. - pub created: u64, - /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. - pub modified: u64, - /// File size in bytes - pub size: u64, -} - -impl Metadata { - /// Create a new metadata struct with the current time as uploaded, created and modified. - pub fn new_with_size(size: u64) -> Self { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(); - - Self { - created: now, - modified: now, - size, - } - } -} - /// Directory structure mapping filepaths to their data maps and metadata. /// /// The data maps are stored within this structure instead of uploading them to the network, keeping the data private. @@ -68,6 +37,14 @@ pub struct PrivateArchive { map: BTreeMap, } +/// This type essentially wraps archive in version marker. E.g. in JSON format: +/// `{ "V0": { "map": } }` +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub enum PrivateArchiveVersioned { + V0(PrivateArchive), +} + impl PrivateArchive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network @@ -94,8 +71,7 @@ impl PrivateArchive { Ok(()) } - /// Add a file to a local archive - /// Note that this does not upload the archive to the network + /// Add a file to a local archive. Note that this does not upload the archive to the network. pub fn add_file(&mut self, path: PathBuf, data_map: DataMapChunk, meta: Metadata) { self.map.insert(path.clone(), (data_map, meta)); debug!("Added a new file to the archive, path: {:?}", path); @@ -109,8 +85,8 @@ impl PrivateArchive { .collect() } - /// List all data addresses of the files in the archive - pub fn addresses(&self) -> Vec { + /// List all data [`DataMapChunk`]s of the files in the archive + pub fn data_maps(&self) -> Vec { self.map .values() .map(|(data_map, _)| data_map.clone()) @@ -133,14 +109,17 @@ impl PrivateArchive { /// Deserialize from bytes. pub fn from_bytes(data: Bytes) -> Result { - let root: PrivateArchive = rmp_serde::from_slice(&data[..])?; + let root: PrivateArchiveVersioned = rmp_serde::from_slice(&data[..])?; + // Currently we have only `V0`. If we add `V1`, then we need an upgrade/migration path here. + let PrivateArchiveVersioned::V0(root) = root; Ok(root) } /// Serialize to bytes. pub fn to_bytes(&self) -> Result { - let root_serialized = rmp_serde::to_vec(&self)?; + let versioned = PrivateArchiveVersioned::V0(self.clone()); + let root_serialized = rmp_serde::to_vec_named(&versioned)?; let root_serialized = Bytes::from(root_serialized); Ok(root_serialized) @@ -151,7 +130,7 @@ impl Client { /// Fetch a [`PrivateArchive`] from the network pub async fn archive_get( &self, - addr: PrivateArchiveAccess, + addr: &PrivateArchiveAccess, ) -> Result { let data = self.data_get(addr).await?; Ok(PrivateArchive::from_bytes(data)?) @@ -162,10 +141,17 @@ impl Client { &self, archive: &PrivateArchive, payment_option: PaymentOption, - ) -> Result { + ) -> Result<(AttoTokens, PrivateArchiveAccess), PutError> { let bytes = archive .to_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; + + #[cfg(feature = "loud")] + println!( + "Uploading private archive referencing {} files", + archive.map().len() + ); + let result = self.data_put(bytes, payment_option).await; debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); result diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/high_level/files/archive_public.rs similarity index 50% rename from autonomi/src/client/files/archive_public.rs rename to autonomi/src/client/high_level/files/archive_public.rs index 4a3a07684a..a879f7e7f2 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/high_level/files/archive_public.rs @@ -11,32 +11,42 @@ use std::{ path::{Path, PathBuf}, }; -use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; +use ant_networking::time::{Duration, SystemTime, UNIX_EPOCH}; -use ant_evm::{AttoTokens, EvmWallet}; +use crate::{AttoTokens, Wallet}; use bytes::Bytes; use serde::{Deserialize, Serialize}; use xor_name::XorName; -use super::archive::Metadata; use crate::{ client::{ - data::{CostError, DataAddr, GetError, PutError}, - files::archive::RenameError, + high_level::{data::DataAddr, files::RenameError}, + quote::CostError, + GetError, PutError, }, Client, }; +use super::Metadata; + /// The address of a public archive on the network. Points to an [`PublicArchive`]. pub type ArchiveAddr = XorName; -/// Public variant of [`crate::client::files::archive::PrivateArchive`]. Differs in that data maps of files are uploaded +/// Public variant of [`crate::client::files::archive_private::PrivateArchive`]. Differs in that data maps of files are uploaded /// to the network, of which the addresses are stored in this archive. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PublicArchive { map: BTreeMap, } +/// This type essentially wraps archive in version marker. E.g. in JSON format: +/// `{ "V0": { "map": } }` +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub enum PublicArchiveVersioned { + V0(PublicArchive), +} + impl PublicArchive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network @@ -46,7 +56,7 @@ impl PublicArchive { } } - /// Rename a file in an archive + /// Rename a file in an archive. /// Note that this does not upload the archive to the network pub fn rename_file(&mut self, old_path: &Path, new_path: &Path) -> Result<(), RenameError> { let (data_addr, mut meta) = self @@ -98,14 +108,17 @@ impl PublicArchive { /// Deserialize from bytes. pub fn from_bytes(data: Bytes) -> Result { - let root: PublicArchive = rmp_serde::from_slice(&data[..])?; + let root: PublicArchiveVersioned = rmp_serde::from_slice(&data[..])?; + // Currently we have only `V0`. If we add `V1`, then we need an upgrade/migration path here. + let PublicArchiveVersioned::V0(root) = root; Ok(root) } /// Serialize to bytes. pub fn to_bytes(&self) -> Result { - let root_serialized = rmp_serde::to_vec(&self)?; + let versioned = PublicArchiveVersioned::V0(self.clone()); + let root_serialized = rmp_serde::to_vec_named(&versioned)?; let root_serialized = Bytes::from(root_serialized); Ok(root_serialized) @@ -122,11 +135,11 @@ impl Client { /// # #[tokio::main] /// # async fn main() -> Result<(), Box> { /// let client = Client::init().await?; - /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; + /// let archive = client.archive_get_public(&ArchiveAddr::random(&mut rand::thread_rng())).await?; /// # Ok(()) /// # } /// ``` - pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { + pub async fn archive_get_public(&self, addr: &ArchiveAddr) -> Result { let data = self.data_get_public(addr).await?; Ok(PublicArchive::from_bytes(data)?) } @@ -138,7 +151,7 @@ impl Client { /// Create simple archive containing `file.txt` pointing to random XOR name. /// /// ```no_run - /// # use autonomi::{Client, client::{data::DataAddr, files::{archive::Metadata, archive_public::{PublicArchive, ArchiveAddr}}}}; + /// # use autonomi::{Client, client::{data::DataAddr, files::{Metadata, archive_public::{PublicArchive, ArchiveAddr}}}}; /// # use std::path::PathBuf; /// # #[tokio::main] /// # async fn main() -> Result<(), Box> { @@ -146,25 +159,32 @@ impl Client { /// # let wallet = todo!(); /// let mut archive = PublicArchive::new(); /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); - /// let address = client.archive_put_public(&archive, &wallet).await?; + /// let (cost, address) = client.archive_put_public(&archive, &wallet).await?; /// # Ok(()) /// # } /// ``` pub async fn archive_put_public( &self, archive: &PublicArchive, - wallet: &EvmWallet, - ) -> Result { + wallet: &Wallet, + ) -> Result<(AttoTokens, ArchiveAddr), PutError> { let bytes = archive .to_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; + + #[cfg(feature = "loud")] + println!( + "Uploading public archive referencing {} files", + archive.map().len() + ); + let result = self.data_put_public(bytes, wallet.into()).await; debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); result } /// Get the cost to upload an archive - pub async fn archive_cost(&self, archive: PublicArchive) -> Result { + pub async fn archive_cost(&self, archive: &PublicArchive) -> Result { let bytes = archive .to_bytes() .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; @@ -173,3 +193,93 @@ impl Client { result } } + +#[cfg(test)] +mod test { + use std::str::FromStr; + + use super::*; + + #[test] + fn compatibility() { + // In the future we'll have an extra variant. + #[derive(Serialize, Deserialize)] + #[non_exhaustive] + pub enum FuturePublicArchiveVersioned { + V0(PublicArchive), + V1(PublicArchive), + #[serde(other)] + Unsupported, + } + + let mut arch = PublicArchive::new(); + arch.add_file( + PathBuf::from_str("hello_world").unwrap(), + DataAddr::random(&mut rand::thread_rng()), + Metadata::new_with_size(1), + ); + let arch_serialized = arch.to_bytes().unwrap(); + + // Create archive, forward compatible (still the same V0 version). + let future_arch = FuturePublicArchiveVersioned::V0(arch.clone()); + let future_arch_serialized = rmp_serde::to_vec_named(&future_arch).unwrap(); + + // Let's see if we can deserialize a (forward compatible) archive arriving to us from the future + let _ = PublicArchive::from_bytes(Bytes::from(future_arch_serialized)).unwrap(); + + // Let's see if we can deserialize an old archive from the future + let _: FuturePublicArchiveVersioned = rmp_serde::from_slice(&arch_serialized[..]).unwrap(); + + // Now we break forward compatibility by introducing a new version not supported by the old code. + let future_arch = FuturePublicArchiveVersioned::V1(arch.clone()); + let future_arch_serialized = rmp_serde::to_vec_named(&future_arch).unwrap(); + // The old archive will not be able to decode this. + assert!(PublicArchive::from_bytes(Bytes::from(future_arch_serialized)).is_err()); + + // Now we prove backwards compatibility. Our old V0 archive will still be decoded by our new archive wrapper as V0. + let versioned_arch = PublicArchiveVersioned::V0(arch.clone()); // 'Old' archive wrapper + let versioned_arch_serialized = rmp_serde::to_vec_named(&versioned_arch).unwrap(); + let _: FuturePublicArchiveVersioned = // Into 'new' wrapper + rmp_serde::from_slice(&versioned_arch_serialized[..]).unwrap(); + } + + #[test] + fn forward_compatibility() { + // What we do here is we create a new `Metadata` and use that in the `Archive` structs. + + /// A version `1.1` which is non-breaking (`1.0` is forward compatible with `1.1`). + #[derive(Debug, Default, Serialize, Deserialize)] + pub struct MetadataV1p1 { + created: u64, + modified: u64, + size: u64, + extra: Option, + accessed: Option, // Added field + } + #[derive(Debug, Default, Serialize, Deserialize)] + pub struct PublicArchiveV1p1 { + map: BTreeMap, + } + #[derive(Debug, Serialize, Deserialize)] + pub enum PublicArchiveVersionedV1p1 { + V0(PublicArchiveV1p1), + } + + let mut arch_p1 = PublicArchiveV1p1::default(); + arch_p1.map.insert( + PathBuf::from_str("hello_world").unwrap(), + ( + DataAddr::random(&mut rand::thread_rng()), + MetadataV1p1 { + accessed: Some(1), + ..Default::default() + }, + ), + ); + let arch_p1_ser = + rmp_serde::to_vec_named(&PublicArchiveVersionedV1p1::V0(arch_p1)).unwrap(); + + // Our old data structure should be forward compatible with the new one. + assert!(PublicArchive::from_bytes(Bytes::from(arch_p1_ser)).is_ok()); + } +} diff --git a/autonomi/src/client/high_level/files/fs_private.rs b/autonomi/src/client/high_level/files/fs_private.rs new file mode 100644 index 0000000000..88998e64be --- /dev/null +++ b/autonomi/src/client/high_level/files/fs_private.rs @@ -0,0 +1,261 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use super::archive_private::{PrivateArchive, PrivateArchiveAccess}; +use super::{get_relative_file_path_from_abs_file_and_folder_path, FILE_UPLOAD_BATCH_SIZE}; +use super::{DownloadError, UploadError}; + +use crate::client::PutError; +use crate::client::{data_types::chunk::DataMapChunk, utils::process_tasks_with_max_concurrency}; +use crate::self_encryption::encrypt; +use crate::{AttoTokens, Client, Wallet}; +use ant_protocol::storage::{Chunk, DataTypes}; +use bytes::Bytes; +use std::path::PathBuf; +use xor_name::XorName; + +impl Client { + /// Download a private file from network to local file system + pub async fn file_download( + &self, + data_access: &DataMapChunk, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let data = self.data_get(data_access).await?; + if let Some(parent) = to_dest.parent() { + tokio::fs::create_dir_all(parent).await?; + debug!("Created parent directories for {to_dest:?}"); + } + tokio::fs::write(to_dest.clone(), data).await?; + debug!("Downloaded file to {to_dest:?}"); + Ok(()) + } + + /// Download a private directory from network to local file system + pub async fn dir_download( + &self, + archive_access: &PrivateArchiveAccess, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let archive = self.archive_get(archive_access).await?; + for (path, addr, _meta) in archive.iter() { + self.file_download(addr, to_dest.join(path)).await?; + } + debug!("Downloaded directory to {to_dest:?}"); + Ok(()) + } + + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// The data maps of these (private) files are not uploaded but returned within the [`PrivateArchive`] return type. + pub async fn dir_upload( + &self, + dir_path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, PrivateArchive), UploadError> { + info!("Uploading directory as private: {dir_path:?}"); + let start = tokio::time::Instant::now(); + + let mut encryption_tasks = vec![]; + + for entry in walkdir::WalkDir::new(&dir_path) { + let entry = entry?; + + if entry.file_type().is_dir() { + continue; + } + + let dir_path = dir_path.clone(); + + encryption_tasks.push(async move { + let file_path = entry.path().to_path_buf(); + + info!("Encrypting file: {file_path:?}.."); + #[cfg(feature = "loud")] + println!("Encrypting file: {file_path:?}.."); + + let data = tokio::fs::read(&file_path) + .await + .map_err(|err| format!("Could not read file {file_path:?}: {err:?}"))?; + let data = Bytes::from(data); + + if data.len() < 3 { + let err_msg = + format!("Skipping file {file_path:?}, as it is smaller than 3 bytes"); + return Err(err_msg); + } + + let now = ant_networking::time::Instant::now(); + + let (data_map_chunk, chunks) = encrypt(data).map_err(|err| err.to_string())?; + + debug!("Encryption of {file_path:?} took: {:.2?}", now.elapsed()); + + let xor_names: Vec<_> = chunks + .iter() + .map(|chunk| (*chunk.name(), chunk.size())) + .collect(); + + let metadata = super::fs_public::metadata_from_entry(&entry); + + let relative_path = + get_relative_file_path_from_abs_file_and_folder_path(&file_path, &dir_path); + + Ok(( + file_path.to_string_lossy().to_string(), + xor_names, + chunks, + (relative_path, DataMapChunk::from(data_map_chunk), metadata), + )) + }); + } + + let mut combined_xor_names: Vec<(XorName, usize)> = vec![]; + let mut combined_chunks: Vec<(String, Vec)> = vec![]; + let mut private_archive = PrivateArchive::new(); + + let encryption_results = + process_tasks_with_max_concurrency(encryption_tasks, *FILE_UPLOAD_BATCH_SIZE).await; + + for encryption_result in encryption_results { + match encryption_result { + Ok((file_path, xor_names, chunked_file, file_data)) => { + info!("Successfully encrypted file: {file_path:?}"); + #[cfg(feature = "loud")] + println!("Successfully encrypted file: {file_path:?}"); + + combined_xor_names.extend(xor_names); + combined_chunks.push((file_path, chunked_file)); + let (relative_path, data_map_chunk, file_metadata) = file_data; + private_archive.add_file(relative_path, data_map_chunk, file_metadata); + } + Err(err_msg) => { + error!("Error during file encryption: {err_msg}"); + } + } + } + + info!("Paying for {} chunks..", combined_xor_names.len()); + #[cfg(feature = "loud")] + println!("Paying for {} chunks..", combined_xor_names.len()); + + let (receipt, skipped_payments_amount) = self + .pay_for_content_addrs( + DataTypes::Chunk, + combined_xor_names.into_iter(), + wallet.into(), + ) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}")) + .map_err(PutError::from)?; + + info!("{skipped_payments_amount} chunks were free"); + + let files_to_upload_amount = combined_chunks.len(); + + let mut upload_tasks = vec![]; + + for (name, chunks) in combined_chunks { + let receipt_clone = receipt.clone(); + + upload_tasks.push(async move { + info!("Uploading file: {name} ({} chunks)..", chunks.len()); + #[cfg(feature = "loud")] + println!("Uploading file: {name} ({} chunks)..", chunks.len()); + + // todo: handle failed uploads + let mut failed_uploads = self + .upload_chunks_with_retries(chunks.iter().collect(), &receipt_clone) + .await; + + let chunks_uploaded = chunks.len() - failed_uploads.len(); + + // Return the last chunk upload error + if let Some(last_chunk_fail) = failed_uploads.pop() { + error!( + "Error uploading chunk ({:?}): {:?}", + last_chunk_fail.0.address(), + last_chunk_fail.1 + ); + + (name, Err(UploadError::from(last_chunk_fail.1))) + } else { + info!("Successfully uploaded {name} ({} chunks)", chunks.len()); + #[cfg(feature = "loud")] + println!("Successfully uploaded {name} ({} chunks)", chunks.len()); + + (name, Ok(chunks_uploaded)) + } + }); + } + + let uploads = + process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; + + info!( + "Upload of {} files completed in {:?}", + files_to_upload_amount, + start.elapsed() + ); + + #[cfg(feature = "loud")] + println!( + "Upload of {} files completed in {:?}", + files_to_upload_amount, + start.elapsed() + ); + + let total_cost = self + .process_upload_results(uploads, receipt, skipped_payments_amount) + .await?; + + Ok((total_cost, private_archive)) + } + + /// Same as [`Client::dir_upload`] but also uploads the archive (privately) to the network. + /// + /// Returns the [`PrivateArchiveAccess`] allowing the private archive to be downloaded from the network. + pub async fn dir_and_archive_upload( + &self, + dir_path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, PrivateArchiveAccess), UploadError> { + let (cost1, archive) = self.dir_upload(dir_path, wallet).await?; + let (cost2, archive_addr) = self.archive_put(&archive, wallet.into()).await?; + let total_cost = cost1.checked_add(cost2).unwrap_or_else(|| { + error!("Total cost overflowed: {cost1:?} + {cost2:?}"); + cost1 + }); + Ok((total_cost, archive_addr)) + } + + /// Upload a private file to the network. + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap) + pub async fn file_upload( + &self, + path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, DataMapChunk), UploadError> { + info!("Uploading file: {path:?}"); + #[cfg(feature = "loud")] + println!("Uploading file: {path:?}"); + + let data = tokio::fs::read(path).await?; + let data = Bytes::from(data); + let (total_cost, addr) = self.data_put(data, wallet.into()).await?; + debug!("Uploaded file successfully in the privateAchive: {addr:?}"); + Ok((total_cost, addr)) + } +} diff --git a/autonomi/src/client/high_level/files/fs_public.rs b/autonomi/src/client/high_level/files/fs_public.rs new file mode 100644 index 0000000000..f4c861516f --- /dev/null +++ b/autonomi/src/client/high_level/files/fs_public.rs @@ -0,0 +1,365 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use super::archive_public::{ArchiveAddr, PublicArchive}; +use super::{DownloadError, FileCostError, Metadata, UploadError}; +use crate::client::high_level::files::{ + get_relative_file_path_from_abs_file_and_folder_path, FILE_UPLOAD_BATCH_SIZE, +}; +use crate::client::{high_level::data::DataAddr, utils::process_tasks_with_max_concurrency}; +use crate::client::{Client, PutError}; +use crate::self_encryption::encrypt; +use crate::{Amount, AttoTokens, Wallet}; +use ant_networking::time::{Duration, SystemTime}; +use ant_protocol::storage::{Chunk, DataTypes}; +use bytes::Bytes; +use std::path::PathBuf; +use xor_name::XorName; + +impl Client { + /// Download file from network to local file system + pub async fn file_download_public( + &self, + data_addr: &DataAddr, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let data = self.data_get_public(data_addr).await?; + if let Some(parent) = to_dest.parent() { + tokio::fs::create_dir_all(parent).await?; + debug!("Created parent directories {parent:?} for {to_dest:?}"); + } + tokio::fs::write(to_dest.clone(), data).await?; + debug!("Downloaded file to {to_dest:?} from the network address {data_addr:?}"); + Ok(()) + } + + /// Download directory from network to local file system + pub async fn dir_download_public( + &self, + archive_addr: &ArchiveAddr, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let archive = self.archive_get_public(archive_addr).await?; + debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); + for (path, addr, _meta) in archive.iter() { + self.file_download_public(addr, to_dest.join(path)).await?; + } + debug!( + "All files in the directory downloaded to {:?} from the network address {:?}", + to_dest.parent(), + archive_addr + ); + Ok(()) + } + + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// + /// The data maps of these files are uploaded on the network, making the individual files publicly available. + /// + /// This returns, but does not upload (!),the [`PublicArchive`] containing the data maps of the uploaded files. + pub async fn dir_upload_public( + &self, + dir_path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, PublicArchive), UploadError> { + info!("Uploading directory: {dir_path:?}"); + let start = tokio::time::Instant::now(); + + let mut encryption_tasks = vec![]; + + for entry in walkdir::WalkDir::new(&dir_path) { + let entry = entry?; + + if entry.file_type().is_dir() { + continue; + } + + let dir_path = dir_path.clone(); + + encryption_tasks.push(async move { + let file_path = entry.path().to_path_buf(); + + info!("Encrypting file: {file_path:?}.."); + #[cfg(feature = "loud")] + println!("Encrypting file: {file_path:?}.."); + + let data = tokio::fs::read(&file_path) + .await + .map_err(|err| format!("Could not read file {file_path:?}: {err:?}"))?; + let data = Bytes::from(data); + + if data.len() < 3 { + let err_msg = + format!("Skipping file {file_path:?}, as it is smaller than 3 bytes"); + return Err(err_msg); + } + + let now = ant_networking::time::Instant::now(); + + let (data_map_chunk, mut chunks) = encrypt(data).map_err(|err| err.to_string())?; + + debug!("Encryption of {file_path:?} took: {:.2?}", now.elapsed()); + + let data_address = *data_map_chunk.name(); + + chunks.push(data_map_chunk); + + let xor_names: Vec<_> = chunks + .iter() + .map(|chunk| (*chunk.name(), chunk.size())) + .collect(); + + let metadata = metadata_from_entry(&entry); + + let relative_path = + get_relative_file_path_from_abs_file_and_folder_path(&file_path, &dir_path); + + Ok(( + file_path.to_string_lossy().to_string(), + xor_names, + chunks, + (relative_path, data_address, metadata), + )) + }); + } + + let mut combined_xor_names: Vec<(XorName, usize)> = vec![]; + let mut combined_chunks: Vec<((String, XorName), Vec)> = vec![]; + let mut public_archive = PublicArchive::new(); + + let encryption_results = + process_tasks_with_max_concurrency(encryption_tasks, *FILE_UPLOAD_BATCH_SIZE).await; + + for encryption_result in encryption_results { + match encryption_result { + Ok((file_path, xor_names, chunks, file_data)) => { + info!("Successfully encrypted file: {file_path:?}"); + #[cfg(feature = "loud")] + println!("Successfully encrypted file: {file_path:?}"); + + combined_xor_names.extend(xor_names); + let (relative_path, data_address, file_metadata) = file_data; + combined_chunks.push(((file_path, data_address), chunks)); + public_archive.add_file(relative_path, data_address, file_metadata); + } + Err(err_msg) => { + error!("Error during file encryption: {err_msg}"); + } + } + } + + info!("Paying for {} chunks..", combined_xor_names.len()); + #[cfg(feature = "loud")] + println!("Paying for {} chunks..", combined_xor_names.len()); + + let (receipt, skipped_payments_amount) = self + .pay_for_content_addrs( + DataTypes::Chunk, + combined_xor_names.into_iter(), + wallet.into(), + ) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}")) + .map_err(PutError::from)?; + + info!("{skipped_payments_amount} chunks were free"); + + let files_to_upload_amount = combined_chunks.len(); + + let mut upload_tasks = vec![]; + + for ((name, data_address), chunks) in combined_chunks { + let receipt_clone = receipt.clone(); + + upload_tasks.push(async move { + info!("Uploading file: {name} ({} chunks)..", chunks.len()); + #[cfg(feature = "loud")] + println!("Uploading file: {name} ({} chunks)..", chunks.len()); + + // todo: handle failed uploads + let mut failed_uploads = self + .upload_chunks_with_retries(chunks.iter().collect(), &receipt_clone) + .await; + + let chunks_uploaded = chunks.len() - failed_uploads.len(); + + // Return the last chunk upload error + if let Some(last_chunk_fail) = failed_uploads.pop() { + error!( + "Error uploading chunk ({:?}): {:?}", + last_chunk_fail.0.address(), + last_chunk_fail.1 + ); + + (name, Err(UploadError::from(last_chunk_fail.1))) + } else { + info!( + "Successfully uploaded {name} ({} chunks) to: {}", + chunks.len(), + hex::encode(data_address.0) + ); + #[cfg(feature = "loud")] + println!( + "Successfully uploaded {name} ({} chunks) to: {}", + chunks.len(), + hex::encode(data_address.0) + ); + + (name, Ok(chunks_uploaded)) + } + }); + } + + let uploads = + process_tasks_with_max_concurrency(upload_tasks, *FILE_UPLOAD_BATCH_SIZE).await; + + info!( + "Upload of {} files completed in {:?}", + files_to_upload_amount, + start.elapsed() + ); + + #[cfg(feature = "loud")] + println!( + "Upload of {} files completed in {:?}", + files_to_upload_amount, + start.elapsed() + ); + + let total_cost = self + .process_upload_results(uploads, receipt, skipped_payments_amount) + .await?; + + Ok((total_cost, public_archive)) + } + + /// Same as [`Client::dir_upload_public`] but also uploads the archive to the network. + /// + /// Returns the [`ArchiveAddr`] of the uploaded archive. + pub async fn dir_and_archive_upload_public( + &self, + dir_path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, ArchiveAddr), UploadError> { + let (cost1, archive) = self.dir_upload_public(dir_path, wallet).await?; + let (cost2, archive_addr) = self.archive_put_public(&archive, wallet).await?; + let total_cost = cost1.checked_add(cost2).unwrap_or_else(|| { + error!("Total cost overflowed: {cost1:?} + {cost2:?}"); + cost1 + }); + Ok((total_cost, archive_addr)) + } + + /// Upload a file to the network. + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) + pub async fn file_upload_public( + &self, + path: PathBuf, + wallet: &Wallet, + ) -> Result<(AttoTokens, DataAddr), UploadError> { + info!("Uploading file: {path:?}"); + #[cfg(feature = "loud")] + println!("Uploading file: {path:?}"); + + let data = tokio::fs::read(path.clone()).await?; + let data = Bytes::from(data); + let (cost, addr) = self.data_put_public(data, wallet.into()).await?; + debug!("File {path:?} uploaded to the network at {addr:?}"); + Ok((cost, addr)) + } + + /// Get the cost to upload a file/dir to the network. + /// quick and dirty implementation, please refactor once files are cleanly implemented + pub async fn file_cost(&self, path: &PathBuf) -> Result { + let mut archive = PublicArchive::new(); + let mut total_cost = Amount::ZERO; + + for entry in walkdir::WalkDir::new(path) { + let entry = entry?; + + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path().to_path_buf(); + tracing::info!("Cost for file: {path:?}"); + + let data = tokio::fs::read(&path).await?; + let file_bytes = Bytes::from(data); + let file_cost = self.data_cost(file_bytes.clone()).await?; + + total_cost += file_cost.as_atto(); + + // re-do encryption to get the correct map xorname here + // this code needs refactor + let now = ant_networking::time::Instant::now(); + let (data_map_chunk, _) = crate::self_encryption::encrypt(file_bytes)?; + tracing::debug!("Encryption took: {:.2?}", now.elapsed()); + let map_xor_name = *data_map_chunk.address().xorname(); + + let metadata = metadata_from_entry(&entry); + archive.add_file(path, map_xor_name, metadata); + } + + let root_serialized = rmp_serde::to_vec(&archive)?; + + let archive_cost = self.data_cost(Bytes::from(root_serialized)).await?; + + total_cost += archive_cost.as_atto(); + debug!("Total cost for the directory: {total_cost:?}"); + Ok(total_cost.into()) + } +} + +// Get metadata from directory entry. Defaults to `0` for creation and modification times if +// any error is encountered. Logs errors upon error. +pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { + let fs_metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(err) => { + tracing::warn!( + "Failed to get metadata for `{}`: {err}", + entry.path().display() + ); + return Metadata { + created: 0, + modified: 0, + size: 0, + extra: None, + }; + } + }; + + let unix_time = |property: &'static str, time: std::io::Result| { + time.inspect_err(|err| { + tracing::warn!( + "Failed to get '{property}' metadata for `{}`: {err}", + entry.path().display() + ); + }) + .unwrap_or(SystemTime::UNIX_EPOCH) + .duration_since(SystemTime::UNIX_EPOCH) + .inspect_err(|err| { + tracing::warn!( + "'{property}' metadata of `{}` is before UNIX epoch: {err}", + entry.path().display() + ); + }) + .unwrap_or(Duration::from_secs(0)) + .as_secs() + }; + let created = unix_time("created", fs_metadata.created()); + let modified = unix_time("modified", fs_metadata.modified()); + + Metadata { + created, + modified, + size: fs_metadata.len(), + extra: None, + } +} diff --git a/autonomi/src/client/high_level/files/fs_shared.rs b/autonomi/src/client/high_level/files/fs_shared.rs new file mode 100644 index 0000000000..65420c37e1 --- /dev/null +++ b/autonomi/src/client/high_level/files/fs_shared.rs @@ -0,0 +1,57 @@ +use crate::client::payment::Receipt; +use crate::client::{ClientEvent, UploadSummary}; +use crate::files::UploadError; +use crate::Client; +use ant_evm::{Amount, AttoTokens}; + +impl Client { + pub(crate) async fn process_upload_results( + &self, + uploads: Vec<(String, Result)>, + receipt: Receipt, + skipped_payments_amount: usize, + ) -> Result { + let mut total_chunks_uploaded = 0; + let mut last_err: Option = None; + + for (name, result) in uploads { + match result { + Ok(chunks_uploaded) => { + total_chunks_uploaded += chunks_uploaded; + } + Err(err) => { + error!("Error uploading file {name}: {err:?}"); + #[cfg(feature = "loud")] + println!("Error uploading file {name}: {err:?}"); + + last_err = Some(err); + } + } + } + + // todo: bundle the errors together in a new error type + // Throw an error if not all files were uploaded successfully + if let Some(err) = last_err { + return Err(err); + } + + let tokens_spent = receipt + .values() + .map(|(_, cost)| cost.as_atto()) + .sum::(); + + // Reporting + if let Some(channel) = self.client_event_sender.as_ref() { + let summary = UploadSummary { + records_paid: total_chunks_uploaded.saturating_sub(skipped_payments_amount), + records_already_paid: skipped_payments_amount, + tokens_spent, + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err:?}"); + } + } + + Ok(AttoTokens::from_atto(tokens_spent)) + } +} diff --git a/autonomi/src/client/high_level/files/mod.rs b/autonomi/src/client/high_level/files/mod.rs new file mode 100644 index 0000000000..36d6fd84e9 --- /dev/null +++ b/autonomi/src/client/high_level/files/mod.rs @@ -0,0 +1,141 @@ +use serde::{Deserialize, Serialize}; +use std::{ + path::{Path, PathBuf}, + sync::LazyLock, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use thiserror::Error; + +use crate::client::{quote::CostError, GetError, PutError}; + +pub mod archive_private; +pub mod archive_public; +pub mod fs_private; +pub mod fs_public; +mod fs_shared; + +pub use archive_private::PrivateArchive; +pub use archive_public::PublicArchive; + +/// Number of files to upload in parallel. +/// +/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. +pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("File upload batch size: {}", batch_size); + batch_size +}); + +/// Metadata for a file in an archive. Time values are UNIX timestamps. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Metadata { + /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. + pub created: u64, + /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. + pub modified: u64, + /// File size in bytes + pub size: u64, + + /// Optional extra metadata with undefined structure, e.g. JSON. + pub extra: Option, +} + +impl Metadata { + /// Create a new metadata struct with the current time as uploaded, created and modified. + pub fn new_with_size(size: u64) -> Self { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + + Self { + created: now, + modified: now, + size, + extra: None, + } + } +} + +#[derive(Error, Debug, PartialEq, Eq)] +pub enum RenameError { + #[error("File not found in archive: {0}")] + FileNotFound(PathBuf), +} + +/// Errors that can occur during the file upload operation. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + #[error("Failed to recursively traverse directory")] + WalkDir(#[from] walkdir::Error), + #[error("Input/output failure")] + IoError(#[from] std::io::Error), + #[error("Failed to upload file")] + PutError(#[from] PutError), + #[error("Failed to fetch file")] + GetError(#[from] GetError), + #[error("Failed to serialize")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), +} + +/// Errors that can occur during the download operation. +#[derive(Debug, thiserror::Error)] +pub enum DownloadError { + #[error("Failed to download file")] + GetError(#[from] GetError), + #[error("IO failure")] + IoError(#[from] std::io::Error), +} + +/// Errors that can occur during the file cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum FileCostError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("IO failure")] + IoError(#[from] std::io::Error), + #[error("Serialization error")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Self encryption error")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Walkdir error")] + WalkDir(#[from] walkdir::Error), +} + +pub(crate) fn get_relative_file_path_from_abs_file_and_folder_path( + abs_file_pah: &Path, + abs_folder_path: &Path, +) -> PathBuf { + // check if the dir is a file + let is_file = abs_folder_path.is_file(); + + // could also be the file name + let dir_name = PathBuf::from( + abs_folder_path + .file_name() + .expect("Failed to get file/dir name"), + ); + + if is_file { + dir_name + } else { + let folder_prefix = abs_folder_path + .parent() + .unwrap_or(Path::new("")) + .to_path_buf(); + abs_file_pah + .strip_prefix(folder_prefix) + .expect("Could not strip prefix path") + .to_path_buf() + } +} diff --git a/autonomi/src/client/high_level/mod.rs b/autonomi/src/client/high_level/mod.rs new file mode 100644 index 0000000000..b246716050 --- /dev/null +++ b/autonomi/src/client/high_level/mod.rs @@ -0,0 +1,29 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +pub mod data; +pub mod files; +pub mod vault; + +/// Registers are a mutable piece of data on the Network. +/// They can be read by anyone and updated only by the register owner. +/// Each entry is signed by the owner and all value history is kept on the Network. +/// They can be accessed on the Network using the RegisterAddress which is effectively the hash of the owner's [`crate::PublicKey`]. +/// This means there can only be one Register per key. +/// +/// The underlying structure of registers is a graph, where each version is a new [`crate::GraphEntry`] +/// Each entry is linked to the previous entry and to the next entry, like a doubly linked list +/// For fast access to the current register value, a [`crate::Pointer`] to the last entry always keeps track of the latest version +/// +/// ```ignore +/// chain of GraphEntry: [register root] <-> [value2] <-> [value3] <-> [latest value] +/// ^ +/// | +/// a Pointer to the latest version: [pointer to head] +/// ``` +pub mod register; diff --git a/autonomi/src/client/high_level/register/history.rs b/autonomi/src/client/high_level/register/history.rs new file mode 100644 index 0000000000..3f21a01653 --- /dev/null +++ b/autonomi/src/client/high_level/register/history.rs @@ -0,0 +1,79 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_networking::{GetRecordError, NetworkError}; + +use crate::client::data_types::graph::{GraphEntryAddress, GraphError}; +use crate::client::high_level::register::{ + PublicKey, RegisterAddress, RegisterError, RegisterValue, +}; +use crate::client::key_derivation::MainPubkey; +use crate::client::Client; + +/// A handle to the register history +#[derive(Clone)] +pub struct RegisterHistory { + client: Client, + register_owner: PublicKey, + current_iter: GraphEntryAddress, +} + +impl RegisterHistory { + fn new(client: Client, register_owner: PublicKey, root: GraphEntryAddress) -> Self { + Self { + client, + register_owner, + current_iter: root, + } + } + + /// Fetch and go to the next register value from the history + /// Returns `Ok(None)` when we reached the end + pub async fn next(&mut self) -> Result, RegisterError> { + let (entry, next_derivation) = match self + .client + .register_get_graph_entry_and_next_derivation_index(&self.current_iter) + .await + { + Ok(res) => res, + Err(RegisterError::GraphError(GraphError::Network(NetworkError::GetRecordError( + GetRecordError::RecordNotFound, + )))) => return Ok(None), + Err(e) => return Err(e), + }; + let next_entry_pk: PublicKey = MainPubkey::from(self.register_owner) + .derive_key(&next_derivation) + .into(); + self.current_iter = GraphEntryAddress::from_owner(next_entry_pk); + Ok(Some(entry.content)) + } + + /// Get all the register values from the history, starting from the first to the latest entry + pub async fn collect(&mut self) -> Result, RegisterError> { + let mut history_from_first = self.clone(); + history_from_first.current_iter = GraphEntryAddress::from_owner(self.register_owner); + let mut values = Vec::new(); + while let Some(value) = history_from_first.next().await? { + values.push(value); + } + Ok(values) + } +} + +impl Client { + /// Get the register history, starting from the root to the latest entry. + /// + /// This returns a [`RegisterHistory`] that can be use to get the register values from the history. + /// + /// [`RegisterHistory::next`] can be used to get the values one by one, from the first to the latest entry. + /// [`RegisterHistory::collect`] can be used to get all the register values from the history from the first to the latest entry. + pub fn register_history(&self, addr: &RegisterAddress) -> RegisterHistory { + let graph_entry_addr = addr.to_underlying_graph_root(); + RegisterHistory::new(self.clone(), addr.owner, graph_entry_addr) + } +} diff --git a/autonomi/src/client/high_level/register/mod.rs b/autonomi/src/client/high_level/register/mod.rs new file mode 100644 index 0000000000..2d5db0858b --- /dev/null +++ b/autonomi/src/client/high_level/register/mod.rs @@ -0,0 +1,390 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::data_types::graph::{GraphContent, GraphEntry, GraphEntryAddress, GraphError}; +use crate::client::data_types::pointer::{PointerAddress, PointerError, PointerTarget}; +use crate::client::key_derivation::{DerivationIndex, MainPubkey, MainSecretKey}; +use crate::client::payment::PaymentOption; +use crate::client::quote::CostError; +use crate::client::Client; +use crate::AttoTokens; +use ant_networking::{GetRecordError, NetworkError}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use xor_name::XorName; + +mod history; + +pub use crate::{PublicKey, SecretKey}; +pub use history::RegisterHistory; + +/// A Register is addressed at a [`RegisterAddress`] which is in fact the owner's [`PublicKey`]. +/// There can only be one register stored at [`PublicKey`]. +/// Any data stored in the register is stored as is, without encryption or modifications. +/// Since the data is publicly accessible by anyone knowing the [`RegisterAddress`], +/// it is up to the owner to encrypt the data uploaded to the register, if wanted. +/// Only the owner can update the register with its [`SecretKey`]. +/// The [`SecretKey`] is the only piece of information an owner should keep to access to the register. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct RegisterAddress { + pub owner: PublicKey, +} + +impl RegisterAddress { + /// Create a new register address + pub fn new(owner: PublicKey) -> Self { + Self { owner } + } + + /// Get the owner of the register + pub fn owner(&self) -> PublicKey { + self.owner + } + + /// To underlying graph representation + pub fn to_underlying_graph_root(&self) -> GraphEntryAddress { + GraphEntryAddress::from_owner(self.owner) + } + + /// Convert a register address to a hex string + pub fn to_hex(&self) -> String { + self.owner.to_hex() + } + + /// Convert a hex string to a register address + pub fn from_hex(hex: &str) -> Result { + let owner = PublicKey::from_hex(hex)?; + Ok(Self { owner }) + } +} + +impl std::fmt::Display for RegisterAddress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_hex()) + } +} + +/// The value of a register: a 32 bytes array (same as [`GraphContent`]) +pub type RegisterValue = GraphContent; + +/// The size of a register value: 32 bytes +pub const REGISTER_VALUE_SIZE: usize = size_of::(); + +#[derive(Error, Debug)] +pub enum RegisterError { + #[error("Underlying GraphError: {0}")] + GraphError(#[from] GraphError), + #[error("Underlying PointerError: {0}")] + PointerError(#[from] PointerError), + #[error("Invalid cost")] + InvalidCost, + #[error("Invalid head pointer, was expecting a GraphEntryAddress but got: {0:?}")] + InvalidHeadPointer(PointerTarget), + #[error("Forked register, this can happen if the register has been updated concurrently, you can solve this by updating the register again with a new value. Concurrent entries: {0:?}")] + Fork(Vec<[u8; 32]>), + #[error("Corrupt register: {0}")] + Corrupt(String), + #[error("Register cannot be updated as it does not exist, please create it first or wait for it to be created")] + CannotUpdateNewRegister, + #[error( + "Invalid register value length: {0}, expected something within {REGISTER_VALUE_SIZE} bytes" + )] + InvalidRegisterValueLength(usize), +} + +/// Hard coded derivation index for the register head pointer +/// Derive the register's main public key by it to get the pointer owner/address +const REGISTER_HEAD_DERIVATION_INDEX: [u8; 32] = [0; 32]; + +impl Client { + /// Create a new register key from a SecretKey and a name. + /// + /// This derives a new [`SecretKey`] from the owner's [`SecretKey`] using the name. + /// Note that you will need to keep track of the names you used to create the register key. + pub fn register_key_from_name(owner: &SecretKey, name: &str) -> SecretKey { + let main_key = MainSecretKey::new(owner.clone()); + let derivation_index = + DerivationIndex::from_bytes(XorName::from_content(name.as_bytes()).0); + main_key.derive_key(&derivation_index).into() + } + + /// Create a new [`RegisterValue`] from bytes, make sure the bytes are not longer than [`REGISTER_VALUE_SIZE`] + pub fn register_value_from_bytes(bytes: &[u8]) -> Result { + if bytes.len() > REGISTER_VALUE_SIZE { + return Err(RegisterError::InvalidRegisterValueLength(bytes.len())); + } + let mut content: RegisterValue = [0; REGISTER_VALUE_SIZE]; + content[..bytes.len()].copy_from_slice(bytes); + Ok(content) + } + + /// Create a new register with an initial value. + /// + /// Note that two payments are required, one for the underlying [`GraphEntry`] and one for the [`crate::Pointer`] + pub async fn register_create( + &self, + owner: &SecretKey, + initial_value: RegisterValue, + payment_option: PaymentOption, + ) -> Result<(AttoTokens, RegisterAddress), RegisterError> { + let main_key = MainSecretKey::new(owner.clone()); + let public_key = main_key.public_key(); + + // create the first entry and decide on the next key + let index = DerivationIndex::random(&mut rand::thread_rng()); + let next_key = main_key.public_key().derive_key(&index); + let parents = vec![]; + let descendants = vec![(next_key.into(), index.into_bytes())]; + let root_entry = GraphEntry::new( + &main_key.clone().into(), + parents, + initial_value, + descendants, + ); + + // put the first entry in the graph + let (graph_cost, addr) = self + .graph_entry_put(root_entry, payment_option.clone()) + .await?; + + // create a Pointer to the last entry + let target = PointerTarget::GraphEntryAddress(addr); + let pointer_key = self.register_head_pointer_sk(&main_key.into()); + let (pointer_cost, _pointer_addr) = self + .pointer_create(&pointer_key, target, payment_option.clone()) + .await?; + let total_cost = graph_cost + .checked_add(pointer_cost) + .ok_or(RegisterError::InvalidCost)?; + Ok(( + total_cost, + RegisterAddress { + owner: public_key.into(), + }, + )) + } + + /// Update the value of a register. + /// + /// The register needs to be created first with [`Client::register_create`] + pub async fn register_update( + &self, + owner: &SecretKey, + new_value: RegisterValue, + payment_option: PaymentOption, + ) -> Result { + // get the pointer of the register head + let addr = RegisterAddress { + owner: owner.public_key(), + }; + let pointer_addr = self.register_head_pointer_address(&addr); + debug!("Getting pointer of register head at {pointer_addr:?}"); + let pointer = match self.pointer_get(&pointer_addr).await { + Ok(pointer) => pointer, + Err(PointerError::Network(NetworkError::GetRecordError( + GetRecordError::RecordNotFound, + ))) => return Err(RegisterError::CannotUpdateNewRegister), + Err(err) => return Err(err.into()), + }; + let graph_entry_addr = match pointer.target() { + PointerTarget::GraphEntryAddress(addr) => addr, + other => return Err(RegisterError::InvalidHeadPointer(other.clone())), + }; + + // get the next derivation index from the current head entry + debug!("Getting register head graph entry at {graph_entry_addr:?}"); + let (parent_entry, new_derivation) = self + .register_get_graph_entry_and_next_derivation_index(graph_entry_addr) + .await?; + + // create a new entry with the new value + let main_key = MainSecretKey::new(owner.clone()); + let new_key = main_key.derive_key(&new_derivation); + let parents = vec![parent_entry.owner]; + let next_derivation = DerivationIndex::random(&mut rand::thread_rng()); + let next_pk = main_key.public_key().derive_key(&next_derivation); + let descendants = vec![(next_pk.into(), next_derivation.into_bytes())]; + let new_entry = GraphEntry::new(&new_key.into(), parents, new_value, descendants); + + // put the new entry in the graph + let (cost, new_graph_entry_addr) = match self + .graph_entry_put(new_entry, payment_option) + .await + { + Ok(res) => res, + Err(GraphError::AlreadyExists(address)) => { + // pointer is apparently not at head, update it + let target = PointerTarget::GraphEntryAddress(address); + let pointer_key = self.register_head_pointer_sk(&main_key.into()); + self.pointer_update(&pointer_key, target).await?; + return Err(RegisterError::Corrupt(format!( + "Pointer is apparently not at head, attempting to heal the register by updating it to point to the next entry at {address:?}, please retry the operation" + ))); + } + Err(err) => return Err(err.into()), + }; + + // update the pointer to point to the new entry + let target = PointerTarget::GraphEntryAddress(new_graph_entry_addr); + let pointer_key = self.register_head_pointer_sk(&main_key.into()); + self.pointer_update(&pointer_key, target).await?; + + Ok(cost) + } + + /// Get the current value of the register + pub async fn register_get( + &self, + addr: &RegisterAddress, + ) -> Result { + // get the pointer of the register head + let pointer_addr = self.register_head_pointer_address(addr); + debug!("Getting pointer of register head at {pointer_addr:?}"); + let pointer = self.pointer_get(&pointer_addr).await?; + let graph_entry_addr = match pointer.target() { + PointerTarget::GraphEntryAddress(addr) => addr, + other => return Err(RegisterError::InvalidHeadPointer(other.clone())), + }; + + // get the entry from the graph + debug!("Getting register head graph entry at {graph_entry_addr:?}"); + let entry = match self.graph_entry_get(graph_entry_addr).await { + Ok(entry) => entry, + Err(GraphError::Fork(entries)) => { + let values = entries.iter().map(|e| e.content).collect::>(); + return Err(RegisterError::Fork(values)); + } + Err(err) => return Err(err.into()), + }; + + // get the content of the entry + let content = entry.content; + Ok(content) + } + + /// Get the cost of a register operation. + /// Returns the cost of creation if it doesn't exist, else returns the cost of an update + pub async fn register_cost(&self, owner: &PublicKey) -> Result { + let pointer_pk = self.register_head_pointer_pk(&RegisterAddress { owner: *owner }); + let graph_entry_cost = self.graph_entry_cost(owner); + let pointer_cost = self.pointer_cost(&pointer_pk); + let (graph_entry_cost, pointer_cost) = + futures::future::join(graph_entry_cost, pointer_cost).await; + graph_entry_cost? + .checked_add(pointer_cost?) + .ok_or(CostError::InvalidCost) + } + + /// Get the address of the register's head pointer + fn register_head_pointer_address(&self, addr: &RegisterAddress) -> PointerAddress { + let pk: MainPubkey = addr.owner.into(); + let pointer_pk = + pk.derive_key(&DerivationIndex::from_bytes(REGISTER_HEAD_DERIVATION_INDEX)); + PointerAddress::from_owner(pointer_pk.into()) + } + + /// Get the secret key of the register's head pointer + fn register_head_pointer_sk(&self, register_owner: &SecretKey) -> SecretKey { + let pointer_sk = MainSecretKey::new(register_owner.clone()) + .derive_key(&DerivationIndex::from_bytes(REGISTER_HEAD_DERIVATION_INDEX)); + pointer_sk.into() + } + + /// Get the public key of the register's head pointer + fn register_head_pointer_pk(&self, addr: &RegisterAddress) -> PublicKey { + let pk: MainPubkey = addr.owner.into(); + let pointer_pk = + pk.derive_key(&DerivationIndex::from_bytes(REGISTER_HEAD_DERIVATION_INDEX)); + pointer_pk.into() + } + + /// Get underlying register graph entry and next derivation index + /// In normal circumstances, there is only one entry with one descendant, yielding ONE entry and ONE derivation index + /// In the case of a fork or a corrupt register, the smallest derivation index among all the entries descendants is chosen + /// We chose here to deal with the errors instead of erroring out to allow users to solve Fork and Corrupt issues by updating the register + async fn register_get_graph_entry_and_next_derivation_index( + &self, + graph_entry_addr: &GraphEntryAddress, + ) -> Result<(GraphEntry, DerivationIndex), RegisterError> { + let entry = match self.graph_entry_get(graph_entry_addr).await { + Ok(e) => e, + Err(GraphError::Fork(entries)) => { + warn!("Forked register, multiple entries found: {entries:?}, choosing the one with the smallest derivation index for the next entry"); + let (entry_by_smallest_derivation, _) = entries + .into_iter() + .filter_map(|e| { + get_derivation_from_graph_entry(&e) + .ok() + .map(|derivation| (e, derivation)) + }) + .min_by(|a, b| a.1.cmp(&b.1)) + .ok_or(RegisterError::Corrupt(format!( + "No valid descendants found for FORKED entry at {graph_entry_addr:?}" + )))?; + entry_by_smallest_derivation + } + Err(err) => return Err(err.into()), + }; + let new_derivation = get_derivation_from_graph_entry(&entry)?; + Ok((entry, new_derivation)) + } +} + +fn get_derivation_from_graph_entry(entry: &GraphEntry) -> Result { + let graph_entry_addr = GraphEntryAddress::from_owner(entry.owner); + let d = match entry.descendants.as_slice() { + [d] => d.1, + _ => return Err(RegisterError::Corrupt(format!( + "Underlying Register GraphEntry at {graph_entry_addr:?} is corrupted, expected one descendant but got {}: {:?}", + entry.descendants.len(), + entry.descendants + ))), + }; + Ok(DerivationIndex::from_bytes(d)) +} + +mod tests { + #[tokio::test] + async fn test_register_by_name() { + let main_key = bls::SecretKey::random(); + let register_key = super::Client::register_key_from_name(&main_key, "register1"); + assert_ne!(register_key.public_key(), main_key.public_key()); + let same_name = super::Client::register_key_from_name(&main_key, "register1"); + assert_eq!(same_name.public_key(), register_key.public_key()); + } + + #[tokio::test] + async fn test_register_value_from_bytes() { + let value = super::Client::register_value_from_bytes(&[1, 2, 3]).unwrap(); + assert_eq!( + value, + [ + 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 + ] + ); + let value = super::Client::register_value_from_bytes(&[ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ]) + .unwrap(); + assert_eq!( + value, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32 + ] + ); + let err = super::Client::register_value_from_bytes(&[ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, + ]) + .unwrap_err(); + assert!(matches!(err, super::RegisterError::InvalidRegisterValueLength(v) if v == 33)); + } +} diff --git a/autonomi/src/client/vault/key.rs b/autonomi/src/client/high_level/vault/key.rs similarity index 85% rename from autonomi/src/client/vault/key.rs rename to autonomi/src/client/high_level/vault/key.rs index e452eddfab..c87669907f 100644 --- a/autonomi/src/client/vault/key.rs +++ b/autonomi/src/client/high_level/vault/key.rs @@ -39,6 +39,17 @@ pub fn derive_vault_key(evm_sk_hex: &str) -> Result Result { + let signature_bytes = hex::decode(signature_hex) + .map_err(|e| VaultKeyError::FailedToGenerateVaultSecretKey(e.to_string()))?; + + let blst_key = derive_secret_key_from_seed(&signature_bytes)?; + let vault_sk = blst_to_blsttc(&blst_key)?; + + Ok(vault_sk) +} + /// Convert a blst secret key to a blsttc secret key and pray that endianness is the same pub(crate) fn blst_to_blsttc(sk: &BlstSecretKey) -> Result { let sk_bytes = sk.to_bytes(); diff --git a/autonomi/src/client/high_level/vault/mod.rs b/autonomi/src/client/high_level/vault/mod.rs new file mode 100644 index 0000000000..fa239b566a --- /dev/null +++ b/autonomi/src/client/high_level/vault/mod.rs @@ -0,0 +1,422 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +pub mod key; +pub mod user_data; + +pub use key::{derive_vault_key, VaultSecretKey}; +pub use user_data::UserData; + +use crate::client::data_types::scratchpad::ScratchpadError; +use crate::client::high_level::files::FILE_UPLOAD_BATCH_SIZE; +use crate::client::key_derivation::{DerivationIndex, MainSecretKey}; +use crate::client::payment::PaymentOption; +use crate::client::quote::CostError; +use crate::client::utils::process_tasks_with_max_concurrency; +use crate::client::Client; +use crate::graph::GraphError; +use ant_evm::{AttoTokens, U256}; +use ant_networking::{GetRecordError, NetworkError}; +use ant_protocol::storage::{ + GraphContent, GraphEntry, GraphEntryAddress, Scratchpad, ScratchpadAddress, +}; +use ant_protocol::Bytes; +use bls::PublicKey; +use std::hash::{DefaultHasher, Hash, Hasher}; +use tracing::info; + +/// The content type of the vault data +/// The number is used to determine the type of the contents of the bytes contained in a vault +/// Custom apps can use this to store their own custom types of data in vaults +/// It is recommended to use the hash of the app name or an unique identifier as the content type using [`app_name_to_vault_content_type`] +/// The value 0 is reserved for tests +pub type VaultContentType = u64; + +/// Defines the max size of content can be written into per ScratchPad +const MAX_CONTENT_PER_SCRATCHPAD: usize = Scratchpad::MAX_SIZE - 1024; + +/// Defines the max number of Scratchpads that one GraphEntry can point to +/// The current value is assuming GraphEntry max_size to be 100KB. +const NUM_OF_SCRATCHPADS_PER_GRAPHENTRY: usize = 1_000; + +/// Hard coded derivation index for the Vault's root GraphEntry. +/// Derive the Vault's main secret/public key by it to get the root GraphEntry owner/address +const VAULT_HEAD_DERIVATION_INDEX: [u8; 32] = [0; 32]; + +/// For custom apps using Vault, this function converts an app identifier or name to a [`VaultContentType`] +pub fn app_name_to_vault_content_type(s: T) -> VaultContentType { + let mut hasher = DefaultHasher::new(); + s.hash(&mut hasher); + hasher.finish() +} + +#[derive(Debug, thiserror::Error)] +pub enum VaultError { + #[error("Vault Scratchpad related error: {0}")] + Scratchpad(#[from] ScratchpadError), + #[error("Vault GraphEntry related error: {0}")] + GraphEntry(#[from] GraphError), + #[error("Vault Cost related error: {0}")] + Cost(#[from] CostError), + #[error("Protocol: {0}")] + Protocol(#[from] ant_protocol::Error), + #[error("Vault doesn't have enough graph descendants: {0}")] + VaultNotEnoughGraphDescendants(String), + #[error("Vault with empty content")] + VaultWithZeroContentSize, +} + +impl Client { + /// Retrieves and returns a decrypted vault if one exists. + /// + /// Returns the content type of the bytes in the vault. + pub async fn fetch_and_decrypt_vault( + &self, + secret_key: &VaultSecretKey, + ) -> Result<(Bytes, VaultContentType), VaultError> { + info!("Fetching and decrypting vault..."); + let main_secret_key = MainSecretKey::new(secret_key.clone()); + let public_key = main_secret_key + .derive_key(&DerivationIndex::from_bytes(VAULT_HEAD_DERIVATION_INDEX)) + .public_key(); + + let mut cur_graph_entry_addr = GraphEntryAddress::from_owner(public_key.into()); + let mut decrypted_full_text = vec![]; + let mut content_type = 0; + let mut has_end_reached = false; + + while !has_end_reached { + let graph_entry = self.graph_entry_get(&cur_graph_entry_addr).await?; + + // The first descendant is reserved for `expand GraphEntry`. + match graph_entry.descendants.split_first() { + Some((&(first, _), rest)) => { + cur_graph_entry_addr = GraphEntryAddress::from_owner(first); + let scratchpad_addresses = rest.to_vec(); + + let (decrypt_data, cur_content_type, is_end_reached) = self + .fetch_scratchpads_of_one_graph_entry_and_decrypt( + &main_secret_key, + scratchpad_addresses, + ) + .await?; + decrypted_full_text.push(decrypt_data); + content_type = cur_content_type; + has_end_reached = is_end_reached; + } + None => { + let msg = format!( + "Vault's GraphEntry at {cur_graph_entry_addr:?} only has {} descendants.", + graph_entry.descendants.len() + ); + return Err(VaultError::VaultNotEnoughGraphDescendants(msg)); + } + } + } + + debug!("vault data is successfully fetched and decrypted"); + Ok((Bytes::from(decrypted_full_text.concat()), content_type)) + } + + /// Get the cost of creating a new vault + /// A quick estimation of cost: + /// num_of_graph_entry * graph_entry_cost + num_of_scratchpad * scratchpad_cost + pub async fn vault_cost( + &self, + owner: &VaultSecretKey, + max_size: u64, + ) -> Result { + if max_size == 0 { + return Err(VaultError::VaultWithZeroContentSize); + } + + info!("Getting cost for vault"); + let public_key = MainSecretKey::new(owner.clone()) + .derive_key(&DerivationIndex::from_bytes(VAULT_HEAD_DERIVATION_INDEX)) + .public_key(); + let graph_entry_cost = self.graph_entry_cost(&public_key.into()).await?; + if graph_entry_cost.is_zero() { + // Has been created, assuming all Scratchpads have been created and paid + Ok(graph_entry_cost) + } else { + let scratchpad_cost = self.scratchpad_cost(&public_key.into()).await?; + + let num_of_scratchpads = max_size / MAX_CONTENT_PER_SCRATCHPAD as u64 + 1; + let num_of_graph_entry = + num_of_scratchpads / NUM_OF_SCRATCHPADS_PER_GRAPHENTRY as u64 + 1; + + let total_cost = U256::from(num_of_graph_entry) * graph_entry_cost.as_atto() + + U256::from(num_of_scratchpads) * scratchpad_cost.as_atto(); + Ok(AttoTokens::from_atto(total_cost)) + } + } + + /// Put data into the client's VaultPacket + /// + /// Dynamically expand the vault capacity by paying for more space (Scratchpad) when needed. + /// + /// It is recommended to use the hash of the app name or unique identifier as the content type. + pub async fn write_bytes_to_vault( + &self, + data: Bytes, + payment_option: PaymentOption, + secret_key: &VaultSecretKey, + content_type: VaultContentType, + ) -> Result { + if data.is_empty() { + return Err(VaultError::VaultWithZeroContentSize); + } + + info!("Writing {} bytes to vault ...", data.len()); + let mut total_cost = AttoTokens::zero(); + let main_secret_key = MainSecretKey::new(secret_key.clone()); + + // scratchpad_derivations ordered by the collection order + let (mut cur_free_graphentry_derivation, mut scratchpad_derivations) = self + .vault_claimed_capacity( + &main_secret_key, + DerivationIndex::from_bytes(VAULT_HEAD_DERIVATION_INDEX), + ) + .await?; + + let contents = split_bytes(data); + + info!( + "Current capacity is {}, meanwhile requiring {}", + scratchpad_derivations.len(), + contents.len() + ); + + // claim more capacity if short of. + // Note: as the Scratchpad is `created on use`, hence during the `claim stage`, + // NUM_OF_SCRATCHPADS_PER_GRAPHENTRY to be claimed in one newly created GraphEntry. + while scratchpad_derivations.len() < contents.len() { + let (new_free_graphentry_derivation, new_scratchpad_derivations, graph_cost) = self + .expand_capacity( + &main_secret_key, + &cur_free_graphentry_derivation, + payment_option.clone(), + ) + .await?; + cur_free_graphentry_derivation = new_free_graphentry_derivation; + scratchpad_derivations.extend(&new_scratchpad_derivations); + total_cost = AttoTokens::from_atto(total_cost.as_atto() + graph_cost.as_atto()); + } + + // Convert to Vec of futures + let update_futures: Vec<_> = contents + .into_iter() + .enumerate() + .map(|(i, content)| { + let sp_secret_key = main_secret_key + .derive_key(&DerivationIndex::from_bytes(scratchpad_derivations[i].1)); + let client = self.clone(); + let payment_option_clone = payment_option.clone(); + + async move { + let target_addr = ScratchpadAddress::new(sp_secret_key.public_key().into()); + let already_exists = self.scratchpad_check_existance(&target_addr).await?; + + if already_exists { + info!( + "Updating Scratchpad at {target_addr:?} with content of {} bytes", + content.len() + ); + match client + .scratchpad_update(&sp_secret_key.clone().into(), content_type, &content) + .await + { + Ok(()) => { + info!( + "Updated Scratchpad at {target_addr:?} with content of {} bytes", + content.len() + ); + Ok(None) + } + Err(err) => Err(err.into()), + } + } else { + info!("Creating Scratchpad at {target_addr:?}"); + let (price, addr) = client + .scratchpad_create( + &sp_secret_key.into(), + content_type, + &content, + payment_option_clone, + ) + .await?; + info!("Created Scratchpad at {addr:?} with cost of {price:?}"); + Ok(Some(price)) + } + } + }) + .collect(); + + let update_results = + process_tasks_with_max_concurrency(update_futures, *FILE_UPLOAD_BATCH_SIZE).await; + + // Process results + for result in update_results { + match result { + Ok(Some(price)) => { + total_cost = AttoTokens::from_atto(total_cost.as_atto() + price.as_atto()); + } + Ok(None) => (), + Err(e) => return Err(e), + } + } + + Ok(total_cost) + } + + // Expand the capacity, i.e. upload one GraphEntry + // The returned value is: + // * cur_free_graphentry_derivation: the output[0] of the tail of the linked GraphEntry + // * scratchpad_derivations: ordered by the creating order + // * graph_cost: cost paid to upload the GraphEntry + async fn expand_capacity( + &self, + main_secret_key: &MainSecretKey, + cur_graphentry_derivation: &DerivationIndex, + payment_option: PaymentOption, + ) -> Result<(DerivationIndex, Vec<(PublicKey, GraphContent)>, AttoTokens), VaultError> { + let own_secret_key = main_secret_key.derive_key(cur_graphentry_derivation); + + // For Vault, doesn't need the backward poining. i.e. one-direction link shall be enough. + let parents = vec![]; + // For Vault, doesn't need this field to be populated. + let initial_value = [0u8; 32]; + + // Poining to the next GraphEntry + let new_graphentry_derivation = DerivationIndex::random(&mut rand::thread_rng()); + let public_key: PublicKey = main_secret_key + .derive_key(&new_graphentry_derivation) + .public_key() + .into(); + let mut descendants = vec![(public_key, new_graphentry_derivation.into_bytes())]; + + // Pointing to other future Scrachpads + descendants.extend((0..NUM_OF_SCRATCHPADS_PER_GRAPHENTRY).map(|_| { + let derivation_index = DerivationIndex::random(&mut rand::thread_rng()); + let public_key: PublicKey = main_secret_key + .derive_key(&derivation_index) + .public_key() + .into(); + (public_key, derivation_index.into_bytes()) + })); + + let graph_entry = GraphEntry::new( + &own_secret_key.into(), + parents, + initial_value, + descendants.clone(), + ); + + // Upload the GraphEntry + let (graph_cost, _addr) = self.graph_entry_put(graph_entry, payment_option).await?; + + let scratchpad_derivations = descendants.split_off(1); + Ok(( + new_graphentry_derivation, + scratchpad_derivations, + graph_cost, + )) + } + + // Collects the current claimed capacity (i.e. the uploaded `GrapthEntry`s) + // The returned value is: + // * cur_free_graphentry_derivation: i.e. the root if no graph_entry uploaded, + // otherwise, the first un-used one (the output[0] of the tail of the linked GraphEntry) + // * scratchpad_derivations: ordered by the collection order + async fn vault_claimed_capacity( + &self, + main_secret_key: &MainSecretKey, + mut cur_free_graphentry_derivation: DerivationIndex, + ) -> Result<(DerivationIndex, Vec<(PublicKey, GraphContent)>), VaultError> { + let mut scratchpad_derivations = vec![]; + loop { + let public_key = main_secret_key + .derive_key(&cur_free_graphentry_derivation) + .public_key(); + let cur_graph_entry_addr = GraphEntryAddress::from_owner(public_key.into()); + + match self.graph_entry_get(&cur_graph_entry_addr).await { + Ok(entry) => { + // A GraphEntry was created with all NUM_OF_SCRATCHPADS_PER_GRAPHENTRY + // scratchpad claimed: + // * the first descendant pointing to next GraphEntry. + // * other descendants pointing to Scratchpads for content. + if entry.descendants.len() <= NUM_OF_SCRATCHPADS_PER_GRAPHENTRY { + let msg = format!("Vault's GraphEntry at {cur_graph_entry_addr:?} only has {} descendants.", + entry.descendants.len()); + return Err(VaultError::VaultNotEnoughGraphDescendants(msg)); + } + cur_free_graphentry_derivation = + DerivationIndex::from_bytes(entry.descendants[0].1); + scratchpad_derivations.extend(&entry.descendants[1..]); + } + Err(GraphError::Network(NetworkError::GetRecordError( + GetRecordError::RecordNotFound, + ))) => { + // GraphEntry not existed, return the current snapshot. + info!( + "vault capacity is successfully fetched, with {} scratchpads", + scratchpad_derivations.len() + ); + return Ok((cur_free_graphentry_derivation, scratchpad_derivations)); + } + Err(err) => { + return Err(err.into()); + } + } + } + } + + async fn fetch_scratchpads_of_one_graph_entry_and_decrypt( + &self, + main_secret_key: &MainSecretKey, + scratchpad_addresses: Vec<(PublicKey, [u8; 32])>, + ) -> Result<(Bytes, VaultContentType, bool), VaultError> { + let mut decrypted_full_text = vec![]; + let mut content_type = 0; + let mut has_end_reached = false; + // Any non-max-sized ScratchPad indicates the end-of-vault-content. + for (pub_key, derive_bytes) in scratchpad_addresses { + let addr = ScratchpadAddress::new(pub_key); + let secret_key = main_secret_key.derive_key(&DerivationIndex::from_bytes(derive_bytes)); + + let sp = self.scratchpad_get(&addr).await?; + content_type = sp.data_encoding(); + let decrypt_data = sp.decrypt_data(&secret_key.into())?; + decrypted_full_text.push(decrypt_data); + if sp.encrypted_data().len() < MAX_CONTENT_PER_SCRATCHPAD { + has_end_reached = true; + break; + } + } + + Ok(( + Bytes::from(decrypted_full_text.concat()), + content_type, + has_end_reached, + )) + } +} + +fn split_bytes(input: Bytes) -> Vec { + let mut contents = Vec::new(); + let mut offset = 0; + + while offset < input.len() { + let end = (offset + MAX_CONTENT_PER_SCRATCHPAD).min(input.len()); + contents.push(input.slice(offset..end)); + offset = end; + } + + contents +} diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/high_level/vault/user_data.rs similarity index 82% rename from autonomi/src/client/vault/user_data.rs rename to autonomi/src/client/high_level/vault/user_data.rs index a0b4069534..ff37495727 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/high_level/vault/user_data.rs @@ -8,20 +8,19 @@ use std::collections::HashMap; -use crate::client::data::GetError; -use crate::client::data::PutError; -use crate::client::files::archive::PrivateArchiveAccess; -use crate::client::files::archive_public::ArchiveAddr; +use crate::client::high_level::files::archive_private::PrivateArchiveAccess; +use crate::client::high_level::files::archive_public::ArchiveAddr; use crate::client::payment::PaymentOption; -use crate::client::registers::RegisterAddress; -use crate::client::vault::VaultError; -use crate::client::vault::{app_name_to_vault_content_type, VaultContentType, VaultSecretKey}; use crate::client::Client; +use crate::client::GetError; +use crate::register::RegisterAddress; use ant_evm::AttoTokens; use ant_protocol::Bytes; use serde::{Deserialize, Serialize}; use std::sync::LazyLock; +use super::{app_name_to_vault_content_type, VaultContentType, VaultError, VaultSecretKey}; + /// Vault content type for UserDataVault pub static USER_DATA_VAULT_CONTENT_IDENTIFIER: LazyLock = LazyLock::new(|| app_name_to_vault_content_type("UserData")); @@ -32,19 +31,17 @@ pub static USER_DATA_VAULT_CONTENT_IDENTIFIER: LazyLock = /// Using User Data Vault is optional, one can decide to keep all their data locally instead. #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct UserData { - /// The register secret key hex encoded - pub register_sk: Option, - /// Owned register addresses, along with their names (can be empty) - pub registers: HashMap, /// Owned file archive addresses, along with their names (can be empty) pub file_archives: HashMap, /// Owned private file archives, along with their names (can be empty) pub private_file_archives: HashMap, + /// Owned register addresses, along with their names (can be empty) + pub register_addresses: HashMap, } /// Errors that can occur during the get operation. #[derive(Debug, thiserror::Error)] -pub enum UserDataVaultGetError { +pub enum UserDataVaultError { #[error("Vault error: {0}")] Vault(#[from] VaultError), #[error("Unsupported vault content type: {0}")] @@ -61,6 +58,11 @@ impl UserData { Self::default() } + /// Add a register. Returning `Option::Some` with the old name if the register was already in the set. + pub fn add_register(&mut self, register: RegisterAddress, name: String) -> Option { + self.register_addresses.insert(register, name) + } + /// Add an archive. Returning `Option::Some` with the old name if the archive was already in the set. pub fn add_file_archive(&mut self, archive: ArchiveAddr) -> Option { self.file_archives.insert(archive, "".into()) @@ -117,35 +119,34 @@ impl Client { pub async fn get_user_data_from_vault( &self, secret_key: &VaultSecretKey, - ) -> Result { + ) -> Result { let (bytes, content_type) = self.fetch_and_decrypt_vault(secret_key).await?; if content_type != *USER_DATA_VAULT_CONTENT_IDENTIFIER { - return Err(UserDataVaultGetError::UnsupportedVaultContentType( + return Err(UserDataVaultError::UnsupportedVaultContentType( content_type, )); } let vault = UserData::from_bytes(bytes).map_err(|e| { - UserDataVaultGetError::Serialization(format!( - "Failed to deserialize vault content: {e}" - )) + UserDataVaultError::Serialization(format!("Failed to deserialize vault content: {e}")) })?; Ok(vault) } /// Put the user data to the vault + /// /// Returns the total cost of the put operation pub async fn put_user_data_to_vault( &self, secret_key: &VaultSecretKey, payment_option: PaymentOption, user_data: UserData, - ) -> Result { - let bytes = user_data - .to_bytes() - .map_err(|e| PutError::Serialization(format!("Failed to serialize user data: {e}")))?; + ) -> Result { + let bytes = user_data.to_bytes().map_err(|e| { + UserDataVaultError::Serialization(format!("Failed to serialize user data: {e}")) + })?; let total_cost = self .write_bytes_to_vault( bytes, diff --git a/autonomi/src/client/key_derivation.rs b/autonomi/src/client/key_derivation.rs new file mode 100644 index 0000000000..39ff89cdaa --- /dev/null +++ b/autonomi/src/client/key_derivation.rs @@ -0,0 +1,350 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use bls::{serde_impl::SerdeSecret, PublicKey, SecretKey, PK_SIZE}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use std::fmt; +use thiserror::Error; + +/// Errors that can occur when decoding a key from a hex string +#[derive(Error, Debug)] +pub enum KeyDecodeError { + #[error("Failed to decode hex to key")] + FailedToDecodeHexToKey, + #[error("Failed to parse BLS key")] + FailedToParseBlsKey, + #[error("Invalid key length")] + InvalidKeyLength, +} + +/// This is used to generate a new DerivedPubkey +/// from a MainPubkey, and the corresponding +/// DerivedSecretKey from the MainSecretKey of that MainPubkey. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] +pub struct DerivationIndex([u8; 32]); + +impl fmt::Debug for DerivationIndex { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!( + formatter, + "{:02x}{:02x}{:02x}..", + self.0[0], self.0[1], self.0[2] + ) + } +} + +impl DerivationIndex { + /// generates a random derivation index + pub fn random(rng: &mut impl RngCore) -> DerivationIndex { + let mut bytes = [0u8; 32]; + rng.fill_bytes(&mut bytes); + DerivationIndex(bytes) + } + + /// returns the inner bytes representation + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + /// returns the inner bytes + pub fn into_bytes(self) -> [u8; 32] { + self.0 + } + + /// Create a new DerivationIndex from a bytes array + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +/// A public key derived from a [`MainPubkey`] using a [`DerivationIndex`] +/// Its associated secret key is the [`DerivedSecretKey`] +/// This key is unlinkable to the original [`MainPubkey`] +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct DerivedPubkey(PublicKey); + +impl DerivedPubkey { + pub fn new>(public_key: G) -> Self { + Self(public_key.into()) + } + + pub fn to_bytes(&self) -> [u8; bls::PK_SIZE] { + self.0.to_bytes() + } + + /// Returns `true` if the signature matches the message. + pub fn verify>(&self, sig: &bls::Signature, msg: M) -> bool { + self.0.verify(sig, msg) + } + + pub fn to_hex(&self) -> String { + hex::encode(self.0.to_bytes()) + } + + pub fn from_hex>(hex: T) -> Result { + let public_key = bls_public_from_hex(hex)?; + Ok(Self::new(public_key)) + } +} + +/// Custom implementation of Serialize and Deserialize for [`DerivedPubkey`] to make it an actionable +/// hex string that can be copy pasted in apps, instead of a useless array of numbers +impl Serialize for DerivedPubkey { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } +} + +impl<'de> Deserialize<'de> for DerivedPubkey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let hex = String::deserialize(deserializer)?; + DerivedPubkey::from_hex(hex).map_err(|e| { + serde::de::Error::custom(format!("Failed to deserialize DerivedPubkey from hex: {e}",)) + }) + } +} + +/// Actionable way to print a DerivedPubkey +/// This way to print it is lengthier but allows to copy/paste it into the cli or other apps +/// To use for verification purposes +impl std::fmt::Debug for DerivedPubkey { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.to_hex()) + } +} + +impl std::fmt::Display for DerivedPubkey { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.to_hex()) + } +} + +/// The secret key of a [`DerivedPubkey`] +/// It is derived from the [`MainSecretKey`] with the same [`DerivationIndex`] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DerivedSecretKey(SerdeSecret); + +impl DerivedSecretKey { + pub fn new>(secret_key: S) -> Self { + Self(SerdeSecret(secret_key.into())) + } + + /// The [`DerivedPubkey`] of this [`DerivedSecretKey`] + pub fn public_key(&self) -> DerivedPubkey { + DerivedPubkey(self.0.public_key()) + } + + /// Sign a message with the secret key + pub fn sign(&self, msg: &[u8]) -> bls::Signature { + self.0.sign(msg) + } +} + +/// This is the public key of the [`MainSecretKey`] +/// One can derive [`DerivedPubkey`]s from this [`MainPubkey`] +#[derive(Copy, PartialEq, Eq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)] +pub struct MainPubkey(pub PublicKey); + +impl MainPubkey { + /// Create a new [`MainPubkey`] from a bls [`PublicKey`] + pub fn new(public_key: PublicKey) -> Self { + Self(public_key) + } + + /// Verify that the signature is valid for the message. + pub fn verify(&self, sig: &bls::Signature, msg: &[u8]) -> bool { + self.0.verify(sig, msg) + } + + /// Generate a new [`DerivedPubkey`] from provided [`DerivationIndex`]. + pub fn derive_key(&self, index: &DerivationIndex) -> DerivedPubkey { + DerivedPubkey(self.0.derive_child(&index.0)) + } + + /// Return the inner pubkey's bytes representation + pub fn to_bytes(self) -> [u8; PK_SIZE] { + self.0.to_bytes() + } + + /// Return a hex representation of the [`MainPubkey`] + pub fn to_hex(&self) -> String { + hex::encode(self.0.to_bytes()) + } + + /// Create a new [`MainPubkey`] from a hex string + pub fn from_hex>(hex: T) -> Result { + let public_key = bls_public_from_hex(hex)?; + Ok(Self::new(public_key)) + } +} + +impl std::fmt::Debug for MainPubkey { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.to_hex()) + } +} + +/// The secret key of the [`MainPubkey`] +/// It is held privately and not shared with anyone +/// With this [`MainSecretKey`], new [`DerivedSecretKey`]:[`DerivedPubkey`] pairs can be generated +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MainSecretKey(SerdeSecret); + +impl MainSecretKey { + /// Create a [`MainSecretKey`] from a bls [`SecretKey`]. + pub fn new(secret_key: SecretKey) -> Self { + Self(SerdeSecret(secret_key)) + } + + /// Return the matching [`MainPubkey`] + pub fn public_key(&self) -> MainPubkey { + MainPubkey(self.0.public_key()) + } + + /// Signs the given message + pub fn sign(&self, msg: &[u8]) -> bls::Signature { + self.0.sign(msg) + } + + /// Derive a [`DerivedSecretKey`] from a [`DerivationIndex`] + /// This is used to create a new unlinkable key pair that cannot be linked back to the [`MainSecretKey`] without the [`DerivationIndex`] + pub fn derive_key(&self, index: &DerivationIndex) -> DerivedSecretKey { + DerivedSecretKey::new(self.0.inner().derive_child(&index.0)) + } + + /// Return the inner secret key's bytes representation + pub fn to_bytes(&self) -> Vec { + self.0.to_bytes().to_vec() + } + + /// Generate a new random [`MainSecretKey`] + pub fn random() -> Self { + Self::new(SecretKey::random()) + } + + /// Generate a new random [`DerivedSecretKey`] from the [`MainSecretKey`] + pub fn random_derived_key(&self, rng: &mut impl RngCore) -> DerivedSecretKey { + self.derive_key(&DerivationIndex::random(rng)) + } +} + +/// Construct a BLS public key from a hex-encoded string. +fn bls_public_from_hex>(hex: T) -> Result { + let bytes = hex::decode(hex).map_err(|_| KeyDecodeError::FailedToDecodeHexToKey)?; + let bytes_fixed_len: [u8; bls::PK_SIZE] = bytes + .as_slice() + .try_into() + .map_err(|_| KeyDecodeError::InvalidKeyLength)?; + let pk = + PublicKey::from_bytes(bytes_fixed_len).map_err(|_| KeyDecodeError::FailedToParseBlsKey)?; + Ok(pk) +} + +// conversions to bls types +impl From for SecretKey { + fn from(main_secret_key: MainSecretKey) -> Self { + main_secret_key.0.inner().to_owned() + } +} +impl From for SecretKey { + fn from(derived_secret_key: DerivedSecretKey) -> Self { + derived_secret_key.0.inner().to_owned() + } +} +impl From for PublicKey { + fn from(derived_pubkey: DerivedPubkey) -> Self { + derived_pubkey.0 + } +} +impl From for PublicKey { + fn from(main_pubkey: MainPubkey) -> Self { + main_pubkey.0 + } +} + +// conversions from bls types +impl From for MainSecretKey { + fn from(secret_key: SecretKey) -> Self { + MainSecretKey::new(secret_key) + } +} +impl From for DerivedSecretKey { + fn from(secret_key: SecretKey) -> Self { + DerivedSecretKey::new(secret_key) + } +} +impl From for MainPubkey { + fn from(public_key: PublicKey) -> Self { + MainPubkey::new(public_key) + } +} +impl From for DerivedPubkey { + fn from(public_key: PublicKey) -> Self { + DerivedPubkey::new(public_key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pubkeys_hex_conversion() -> eyre::Result<()> { + let sk = bls::SecretKey::random(); + let pk = sk.public_key(); + let main_pubkey = MainPubkey::new(pk); + let unique_pubkey = + main_pubkey.derive_key(&DerivationIndex::random(&mut rand::thread_rng())); + + let main_pubkey_hex = main_pubkey.to_hex(); + let unique_pubkey_hex = unique_pubkey.to_hex(); + + let main_pubkey_from_hex = MainPubkey::from_hex(main_pubkey_hex)?; + let unique_pubkey_from_hex = DerivedPubkey::from_hex(unique_pubkey_hex)?; + + assert_eq!(main_pubkey, main_pubkey_from_hex); + assert_eq!(unique_pubkey, unique_pubkey_from_hex); + Ok(()) + } + + #[test] + fn test_serialisation() -> eyre::Result<()> { + let pk = SecretKey::random().public_key(); + let main_pubkey = MainPubkey::new(pk); + let unique_pk = main_pubkey.derive_key(&DerivationIndex::random(&mut rand::thread_rng())); + + let str_serialised = rmp_serde::to_vec_named(&unique_pk)?; + let str_deserialised: DerivedPubkey = rmp_serde::from_slice(&str_serialised)?; + assert_eq!(str_deserialised, unique_pk); + + Ok(()) + } + + #[test] + fn verification_using_child_key() -> eyre::Result<()> { + let msg = "just a test string".as_bytes(); + let main_sk = MainSecretKey::random(); + let derived_sk = main_sk.random_derived_key(&mut rand::thread_rng()); + + // Signature signed by parent key can not be verified by the child key. + let signature = main_sk.sign(msg); + assert!(main_sk.public_key().verify(&signature, msg)); + assert!(!derived_sk.public_key().verify(&signature, msg)); + + // Signature signed by child key can not be verified by the parent key. + let signature = derived_sk.sign(msg); + assert!(derived_sk.public_key().verify(&signature, msg)); + assert!(!main_sk.public_key().verify(&signature, msg)); + + Ok(()) + } +} diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 699a98703f..602c60b030 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -9,39 +9,51 @@ // Optionally enable nightly `doc_cfg`. Allows items to be annotated, e.g.: "Available on crate feature X only". #![cfg_attr(docsrs, feature(doc_cfg))] +/// The 4 basic Network data types. +/// - Chunk +/// - GraphEntry +/// - Pointer +/// - Scratchpad +pub mod data_types; +pub use data_types::chunk; +pub use data_types::graph; +pub use data_types::pointer; +pub use data_types::scratchpad; + +/// High-level types built on top of the basic Network data types. +/// Includes data, files and personnal data vaults +mod high_level; +pub use high_level::data; +pub use high_level::files; +pub use high_level::register; +pub use high_level::vault; + pub mod address; +pub mod config; +pub mod key_derivation; pub mod payment; pub mod quote; -pub mod data; -pub mod files; -pub mod transactions; - #[cfg(feature = "external-signer")] #[cfg_attr(docsrs, doc(cfg(feature = "external-signer")))] pub mod external_signer; -#[cfg(feature = "registers")] -#[cfg_attr(docsrs, doc(cfg(feature = "registers")))] -pub mod registers; -#[cfg(feature = "vault")] -#[cfg_attr(docsrs, doc(cfg(feature = "vault")))] -pub mod vault; - -#[cfg(target_arch = "wasm32")] -pub mod wasm; // private module with utility functions -mod rate_limiter; mod utils; use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore, PeersArgs}; pub use ant_evm::Amount; use ant_evm::EvmNetwork; -use ant_networking::{interval, multiaddr_is_global, Network, NetworkBuilder, NetworkEvent}; -use ant_protocol::version::IDENTIFY_PROTOCOL_STR; +use ant_networking::{ + interval, multiaddr_is_global, Network, NetworkBuilder, NetworkError, NetworkEvent, +}; +use ant_protocol::{version::IDENTIFY_PROTOCOL_STR, NetworkAddress}; +use config::{ClientConfig, ClientOperatingStrategy}; use libp2p::{identity::Keypair, Multiaddr}; -use std::{collections::HashSet, sync::Arc, time::Duration}; -use tokio::sync::mpsc; +use payment::PayError; +use quote::CostError; +use std::{collections::HashSet, time::Duration}; +use tokio::sync::{mpsc, watch}; /// Time before considering the connection timed out. pub const CONNECT_TIMEOUT_SECS: u64 = 10; @@ -68,34 +80,13 @@ pub use ant_protocol::CLOSE_GROUP_SIZE; #[derive(Clone)] pub struct Client { pub(crate) network: Network, - pub(crate) client_event_sender: Arc>>, - pub(crate) evm_network: EvmNetwork, -} - -/// Configuration for [`Client::init_with_config`]. -#[derive(Debug, Clone)] -pub struct ClientConfig { - /// Whether we're expected to connect to a local network. - /// - /// If `local` feature is enabled, [`ClientConfig::default()`] will set this to `true`. - pub local: bool, - - /// List of peers to connect to. - /// - /// If not provided, the client will use the default bootstrap peers. - pub peers: Option>, -} - -impl Default for ClientConfig { - fn default() -> Self { - Self { - #[cfg(feature = "local")] - local: true, - #[cfg(not(feature = "local"))] - local: false, - peers: None, - } - } + pub(crate) client_event_sender: Option>, + /// The EVM network to use for the client. + evm_network: EvmNetwork, + /// The configuration for operations on the client. + config: ClientOperatingStrategy, + // Shutdown signal for child tasks. Sends signal when dropped. + _shutdown_tx: watch::Sender, } /// Error returned by [`Client::init`]. @@ -110,10 +101,48 @@ pub enum ConnectError { TimedOutWithIncompatibleProtocol(HashSet, String), /// An error occurred while bootstrapping the client. - #[error("Failed to bootstrap the client")] + #[error("Failed to bootstrap the client: {0}")] Bootstrap(#[from] ant_bootstrap::Error), } +/// Errors that can occur during the put operation. +#[derive(Debug, thiserror::Error)] +pub enum PutError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("A network error occurred.")] + Network(#[from] NetworkError), + #[error("Error occurred during cost estimation.")] + CostError(#[from] CostError), + #[error("Error occurred during payment.")] + PayError(#[from] PayError), + #[error("Serialization error: {0}")] + Serialization(String), + #[error("A wallet error occurred.")] + Wallet(#[from] ant_evm::EvmError), + #[error("The owner key does not match the client's public key")] + ScratchpadBadOwner, + #[error("Payment unexpectedly invalid for {0:?}")] + PaymentUnexpectedlyInvalid(NetworkAddress), + #[error("The payment proof contains no payees.")] + PayeesMissing, +} + +/// Errors that can occur during the get operation. +#[derive(Debug, thiserror::Error)] +pub enum GetError { + #[error("Could not deserialize data map.")] + InvalidDataMap(rmp_serde::decode::Error), + #[error("Failed to decrypt data.")] + Decryption(crate::self_encryption::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), + #[error("General networking error: {0:?}")] + Network(#[from] NetworkError), + #[error("General protocol error: {0:?}")] + Protocol(#[from] ant_protocol::Error), +} + impl Client { /// Initialize the client with default configuration. /// @@ -126,11 +155,7 @@ impl Client { /// /// See [`Client::init_with_config`]. pub async fn init_local() -> Result { - Self::init_with_config(ClientConfig { - local: true, - ..Default::default() - }) - .await + Self::init_with_config(ClientConfig::local(None)).await } /// Initialize a client that bootstraps from a list of peers. @@ -153,6 +178,8 @@ impl Client { Self::init_with_config(ClientConfig { local, peers: Some(peers), + evm_network: EvmNetwork::new(local).unwrap_or_default(), + strategy: Default::default(), }) .await } @@ -172,11 +199,12 @@ impl Client { /// # } /// ``` pub async fn init_with_config(config: ClientConfig) -> Result { - let (network, event_receiver) = build_client_and_run_swarm(config.local); + let (shutdown_tx, network, event_receiver) = build_client_and_run_swarm(config.local); let peers_args = PeersArgs { disable_mainnet_contacts: config.local, addrs: config.peers.unwrap_or_default(), + local: config.local, ..Default::default() }; @@ -187,7 +215,7 @@ impl Client { let network_clone = network.clone(); let peers = peers.to_vec(); - let _handle = ant_networking::target_arch::spawn(async move { + let _handle = ant_networking::time::spawn(async move { for addr in peers { if let Err(err) = network_clone.dial(addr.clone()).await { error!("Failed to dial addr={addr} with err: {err:?}"); @@ -197,69 +225,20 @@ impl Client { // Wait until we have added a few peers to our routing table. let (sender, receiver) = futures::channel::oneshot::channel(); - ant_networking::target_arch::spawn(handle_event_receiver(event_receiver, sender)); + ant_networking::time::spawn(handle_event_receiver( + event_receiver, + sender, + shutdown_tx.subscribe(), + )); receiver.await.expect("sender should not close")?; debug!("Enough peers were added to our routing table, initialization complete"); Ok(Self { network, - client_event_sender: Arc::new(None), - evm_network: Default::default(), - }) - } - - /// Connect to the network. - /// - /// This will timeout after [`CONNECT_TIMEOUT_SECS`] secs. - /// - /// ```no_run - /// # use autonomi::client::Client; - /// # #[tokio::main] - /// # async fn main() -> Result<(), Box> { - /// let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; - /// #[allow(deprecated)] - /// let client = Client::connect(&peers).await?; - /// # Ok(()) - /// # } - /// ``` - #[deprecated( - since = "0.2.4", - note = "Use [`Client::init`] or [`Client::init_with_config`] instead" - )] - pub async fn connect(peers: &[Multiaddr]) -> Result { - // Any global address makes the client non-local - let local = !peers.iter().any(multiaddr_is_global); - - let (network, event_receiver) = build_client_and_run_swarm(local); - - // Spawn task to dial to the given peers - let network_clone = network.clone(); - let peers = peers.to_vec(); - let _handle = ant_networking::target_arch::spawn(async move { - for addr in peers { - if let Err(err) = network_clone.dial(addr.clone()).await { - error!("Failed to dial addr={addr} with err: {err:?}"); - eprintln!("addr={addr} Failed to dial: {err:?}"); - }; - } - }); - - let (sender, receiver) = futures::channel::oneshot::channel(); - ant_networking::target_arch::spawn(handle_event_receiver(event_receiver, sender)); - - receiver.await.expect("sender should not close")?; - debug!("Client is connected to the network"); - - // With the switch to the new bootstrap cache scheme, - // Seems the too many `initial dial`s could result in failure, - // when startup quoting/upload tasks got started up immediatly. - // Hence, put in a forced wait to allow `initial network discovery` to be completed. - ant_networking::target_arch::sleep(Duration::from_secs(5)).await; - - Ok(Self { - network, - client_event_sender: Arc::new(None), - evm_network: Default::default(), + client_event_sender: None, + evm_network: config.evm_network, + config: config.strategy, + _shutdown_tx: shutdown_tx, }) } @@ -267,21 +246,23 @@ impl Client { pub fn enable_client_events(&mut self) -> mpsc::Receiver { let (client_event_sender, client_event_receiver) = tokio::sync::mpsc::channel(CLIENT_EVENT_CHANNEL_SIZE); - self.client_event_sender = Arc::new(Some(client_event_sender)); + self.client_event_sender = Some(client_event_sender); debug!("All events to the clients are enabled"); client_event_receiver } - pub fn set_evm_network(&mut self, evm_network: EvmNetwork) { - self.evm_network = evm_network; + pub fn evm_network(&self) -> &EvmNetwork { + &self.evm_network } } -fn build_client_and_run_swarm(local: bool) -> (Network, mpsc::Receiver) { +fn build_client_and_run_swarm( + local: bool, +) -> (watch::Sender, Network, mpsc::Receiver) { let mut network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), local); - if let Ok(mut config) = BootstrapCacheConfig::default_config() { + if let Ok(mut config) = BootstrapCacheConfig::default_config(local) { if local { config.disable_cache_writing = true; } @@ -292,30 +273,43 @@ fn build_client_and_run_swarm(local: bool) -> (Network, mpsc::Receiver` from `ant-networking`. Else users need to keep their `tokio` dependency in sync. // TODO: Think about handling the mDNS error here. - let (network, event_receiver, swarm_driver) = - network_builder.build_client().expect("mdns to succeed"); + let (network, event_receiver, swarm_driver) = network_builder.build_client(); + + // TODO: Implement graceful SwarmDriver shutdown for client. + // Create a shutdown signal channel + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let _swarm_driver = ant_networking::time::spawn(swarm_driver.run(shutdown_rx)); - let _swarm_driver = ant_networking::target_arch::spawn(swarm_driver.run()); debug!("Client swarm driver is running"); - (network, event_receiver) + (shutdown_tx, network, event_receiver) } async fn handle_event_receiver( mut event_receiver: mpsc::Receiver, sender: futures::channel::oneshot::Sender>, + mut shutdown_rx: watch::Receiver, ) { // We switch this to `None` when we've sent the oneshot 'connect' result. let mut sender = Some(sender); let mut unsupported_protocols = vec![]; let mut timeout_timer = interval(Duration::from_secs(CONNECT_TIMEOUT_SECS)); - - #[cfg(not(target_arch = "wasm32"))] timeout_timer.tick().await; loop { tokio::select! { + // polls futures in order they appear here (as opposed to random) + biased; + + // Check for a shutdown command. + result = shutdown_rx.changed() => { + if result.is_ok() && *shutdown_rx.borrow() || result.is_err() { + info!("Shutdown signal received or sender dropped. Exiting event receiver loop."); + break; + } + } _ = timeout_timer.tick() => { if let Some(sender) = sender.take() { if unsupported_protocols.len() > 1 { @@ -371,6 +365,10 @@ pub enum ClientEvent { /// Summary of an upload operation. #[derive(Debug, Clone)] pub struct UploadSummary { - pub record_count: usize, + /// Records that were uploaded to the network + pub records_paid: usize, + /// Records that were already paid for so were not re-uploaded + pub records_already_paid: usize, + /// Total cost of the upload pub tokens_spent: Amount, } diff --git a/autonomi/src/client/payment.rs b/autonomi/src/client/payment.rs index 29a8f11576..d5804e054c 100644 --- a/autonomi/src/client/payment.rs +++ b/autonomi/src/client/payment.rs @@ -1,13 +1,33 @@ -use crate::client::data::PayError; -use crate::client::quote::StoreQuote; +use crate::client::quote::{DataTypes, StoreQuote}; use crate::Client; -use ant_evm::{AttoTokens, EncodedPeerId, EvmWallet, ProofOfPayment}; +use ant_evm::{EncodedPeerId, EvmWallet, EvmWalletError, ProofOfPayment}; use std::collections::HashMap; use xor_name::XorName; +use super::quote::CostError; + +pub use crate::{Amount, AttoTokens}; + /// Contains the proof of payments for each XOR address and the amount paid pub type Receipt = HashMap; +pub type AlreadyPaidAddressesCount = usize; + +/// Errors that can occur during the pay operation. +#[derive(Debug, thiserror::Error)] +pub enum PayError { + #[error( + "EVM wallet and client use different EVM networks. Please use the same network for both." + )] + EvmWalletNetworkMismatch, + #[error("Wallet error: {0:?}")] + EvmWalletError(#[from] EvmWalletError), + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Cost error: {0:?}")] + Cost(#[from] CostError), +} + pub fn receipt_from_store_quotes(quotes: StoreQuote) -> Receipt { let mut receipt = Receipt::new(); @@ -38,7 +58,9 @@ pub fn receipt_from_store_quotes(quotes: StoreQuote) -> Receipt { /// Payment options for data payments. #[derive(Clone)] pub enum PaymentOption { + /// Pay using an evm wallet Wallet(EvmWallet), + /// When data was already paid for, use the receipt Receipt(Receipt), } @@ -63,15 +85,62 @@ impl From for PaymentOption { impl Client { pub(crate) async fn pay_for_content_addrs( &self, - content_addrs: impl Iterator + Clone, + data_type: DataTypes, + content_addrs: impl Iterator + Clone, payment_option: PaymentOption, - ) -> Result { + ) -> Result<(Receipt, AlreadyPaidAddressesCount), PayError> { match payment_option { PaymentOption::Wallet(wallet) => { - let receipt = self.pay(content_addrs, &wallet).await?; - Ok(receipt) + let (receipt, skipped) = self.pay(data_type, content_addrs, &wallet).await?; + Ok((receipt, skipped)) } - PaymentOption::Receipt(receipt) => Ok(receipt), + PaymentOption::Receipt(receipt) => Ok((receipt, 0)), + } + } + + /// Pay for the chunks and get the proof of payment. + pub(crate) async fn pay( + &self, + data_type: DataTypes, + content_addrs: impl Iterator + Clone, + wallet: &EvmWallet, + ) -> Result<(Receipt, AlreadyPaidAddressesCount), PayError> { + // Check if the wallet uses the same network as the client + if wallet.network() != self.evm_network() { + return Err(PayError::EvmWalletNetworkMismatch); } + + let number_of_content_addrs = content_addrs.clone().count(); + let quotes = self.get_store_quotes(data_type, content_addrs).await?; + + if !quotes.is_empty() { + // Make sure nobody else can use the wallet while we are paying + debug!("Waiting for wallet lock"); + let lock_guard = wallet.lock().await; + debug!("Locked wallet"); + + // TODO: the error might contain some succeeded quote payments as well. These should be returned on err, so that they can be skipped when retrying. + // TODO: retry when it fails? + // Execute chunk payments + let _payments = wallet + .pay_for_quotes(quotes.payments()) + .await + .map_err(|err| PayError::from(err.0))?; + + // payment is done, unlock the wallet for other threads + drop(lock_guard); + debug!("Unlocked wallet"); + } + + let skipped_chunks = number_of_content_addrs - quotes.len(); + trace!( + "Chunk payments of {} chunks completed. {} chunks were free / already paid for", + quotes.len(), + skipped_chunks + ); + + let receipt = receipt_from_store_quotes(quotes); + + Ok((receipt, skipped_chunks)) } } diff --git a/autonomi/src/client/quote.rs b/autonomi/src/client/quote.rs index 38dfd7f6fd..0c8e8df972 100644 --- a/autonomi/src/client/quote.rs +++ b/autonomi/src/client/quote.rs @@ -6,16 +6,24 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use super::{data::CostError, Client}; -use crate::client::rate_limiter::RateLimiter; +use super::Client; +use crate::client::high_level::files::FILE_UPLOAD_BATCH_SIZE; +use crate::client::utils::process_tasks_with_max_concurrency; use ant_evm::payment_vault::get_market_price; -use ant_evm::{Amount, EvmNetwork, PaymentQuote, QuotePayment, QuotingMetrics}; +use ant_evm::{Amount, PaymentQuote, QuotePayment, QuotingMetrics}; use ant_networking::{Network, NetworkError}; use ant_protocol::{storage::ChunkAddress, NetworkAddress, CLOSE_GROUP_SIZE}; use libp2p::PeerId; use std::collections::HashMap; use xor_name::XorName; +pub use ant_protocol::storage::DataTypes; + +// todo: limit depends per RPC endpoint. We should make this configurable +// todo: test the limit for the Arbitrum One public RPC endpoint +// Working limit of the Arbitrum Sepolia public RPC endpoint +const GET_MARKET_PRICE_BATCH_LIMIT: usize = 2000; + /// A quote for a single address pub struct QuoteForAddress(pub(crate) Vec<(PeerId, PaymentQuote, Amount)>); @@ -52,83 +60,131 @@ impl StoreQuote { } } +/// Errors that can occur during the cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum CostError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Could not get store quote for: {0:?} after several retries")] + CouldNotGetStoreQuote(XorName), + #[error("Could not get store costs: {0:?}")] + CouldNotGetStoreCosts(NetworkError), + #[error("Not enough node quotes for {0:?}, got: {1:?} and need at least {2:?}")] + NotEnoughNodeQuotes(XorName, usize, usize), + #[error("Failed to serialize {0}")] + Serialization(String), + #[error("Market price error: {0:?}")] + MarketPriceError(#[from] ant_evm::payment_vault::error::Error), + #[error("Received invalid cost")] + InvalidCost, +} + impl Client { - pub(crate) async fn get_store_quotes( + pub async fn get_store_quotes( &self, - content_addrs: impl Iterator, + data_type: DataTypes, + content_addrs: impl Iterator, ) -> Result { - // get all quotes from nodes let futures: Vec<_> = content_addrs .into_iter() - .map(|content_addr| fetch_store_quote_with_retries(&self.network, content_addr)) + .map(|(content_addr, data_size)| { + fetch_store_quote_with_retries( + &self.network, + content_addr, + data_type.get_index(), + data_size, + ) + }) .collect(); - let raw_quotes_per_addr = futures::future::try_join_all(futures).await?; - // choose the quotes to pay for each address - let mut quotes_to_pay_per_addr = HashMap::new(); + let raw_quotes_per_addr = + process_tasks_with_max_concurrency(futures, *FILE_UPLOAD_BATCH_SIZE).await; - let mut rate_limiter = RateLimiter::new(); + let mut all_quotes = Vec::new(); + + for result in raw_quotes_per_addr { + let (content_addr, mut raw_quotes) = result?; + debug!( + "fetched raw quotes for content_addr: {content_addr}, with {} quotes.", + raw_quotes.len() + ); - for (content_addr, raw_quotes) in raw_quotes_per_addr { - // FIXME: find better way to deal with paid content addrs and feedback to the user - // assume that content addr is already paid for and uploaded if raw_quotes.is_empty() { + debug!("content_addr: {content_addr} is already paid for. No need to fetch market price."); continue; } - // ask smart contract for the market price - let quoting_metrics: Vec = raw_quotes - .clone() + let target_addr = NetworkAddress::from_chunk_address(ChunkAddress::new(content_addr)); + + // Only keep the quotes of the 5 closest nodes + raw_quotes.sort_by_key(|(peer_id, _)| { + NetworkAddress::from_peer(*peer_id).distance(&target_addr) + }); + raw_quotes.truncate(CLOSE_GROUP_SIZE); + + for (peer_id, quote) in raw_quotes.into_iter() { + all_quotes.push((content_addr, peer_id, quote)); + } + } + + let mut all_prices = Vec::new(); + + for chunk in all_quotes.chunks(GET_MARKET_PRICE_BATCH_LIMIT) { + let quoting_metrics: Vec = chunk .iter() - .map(|(_, q)| q.quoting_metrics.clone()) + .map(|(_, _, quote)| quote.quoting_metrics.clone()) .collect(); - let all_prices = get_market_price_with_rate_limiter_and_retries( - &self.evm_network, - &mut rate_limiter, - quoting_metrics.clone(), - ) - .await?; - - let mut prices: Vec<(PeerId, PaymentQuote, Amount)> = all_prices - .into_iter() - .zip(raw_quotes.into_iter()) - .map(|(price, (peer, quote))| (peer, quote, price)) - .collect(); + debug!( + "Getting market prices for {} quoting metrics", + quoting_metrics.len() + ); - // sort by price - prices.sort_by(|(_, _, price_a), (_, _, price_b)| price_a.cmp(price_b)); - - // we need at least 5 valid quotes to pay for the data - const MINIMUM_QUOTES_TO_PAY: usize = 5; - match &prices[..] { - [first, second, third, fourth, fifth, ..] => { - let (p1, q1, _) = first; - let (p2, q2, _) = second; - - // don't pay for the cheapest 2 quotes but include them - let first = (*p1, q1.clone(), Amount::ZERO); - let second = (*p2, q2.clone(), Amount::ZERO); - - // pay for the rest - quotes_to_pay_per_addr.insert( - content_addr, - QuoteForAddress(vec![ - first, - second, - third.clone(), - fourth.clone(), - fifth.clone(), - ]), - ); - } - _ => { - return Err(CostError::NotEnoughNodeQuotes( - content_addr, - prices.len(), - MINIMUM_QUOTES_TO_PAY, - )); - } + let batch_prices = get_market_price(&self.evm_network, quoting_metrics).await?; + + all_prices.extend(batch_prices); + } + + let quotes_with_prices: Vec<(XorName, PeerId, PaymentQuote, Amount)> = all_quotes + .into_iter() + .zip(all_prices.into_iter()) + .map(|((content_addr, peer_id, quote), price)| (content_addr, peer_id, quote, price)) + .collect(); + + let mut quotes_per_addr: HashMap> = + HashMap::new(); + + for (content_addr, peer_id, quote, price) in quotes_with_prices { + let entry = quotes_per_addr.entry(content_addr).or_default(); + entry.push((peer_id, quote, price)); + entry.sort_by_key(|(_, _, price)| *price); + } + + let mut quotes_to_pay_per_addr = HashMap::new(); + + const MINIMUM_QUOTES_TO_PAY: usize = 5; + + for (content_addr, quotes) in quotes_per_addr { + if quotes.len() >= MINIMUM_QUOTES_TO_PAY { + let (p1, q1, _) = "es[0]; + let (p2, q2, _) = "es[1]; + + quotes_to_pay_per_addr.insert( + content_addr, + QuoteForAddress(vec![ + (*p1, q1.clone(), Amount::ZERO), + (*p2, q2.clone(), Amount::ZERO), + quotes[2].clone(), + quotes[3].clone(), + quotes[4].clone(), + ]), + ); + } else { + return Err(CostError::NotEnoughNodeQuotes( + content_addr, + quotes.len(), + MINIMUM_QUOTES_TO_PAY, + )); } } @@ -140,10 +196,14 @@ impl Client { async fn fetch_store_quote( network: &Network, content_addr: XorName, + data_type: u32, + data_size: usize, ) -> Result, NetworkError> { network .get_store_quote_from_network( NetworkAddress::from_chunk_address(ChunkAddress::new(content_addr)), + data_type, + data_size, vec![], ) .await @@ -153,12 +213,18 @@ async fn fetch_store_quote( async fn fetch_store_quote_with_retries( network: &Network, content_addr: XorName, + data_type: u32, + data_size: usize, ) -> Result<(XorName, Vec<(PeerId, PaymentQuote)>), CostError> { let mut retries = 0; loop { - match fetch_store_quote(network, content_addr).await { + match fetch_store_quote(network, content_addr, data_type, data_size).await { Ok(quote) => { + if quote.is_empty() { + // Empty quotes indicates the record already exists. + break Ok((content_addr, quote)); + } if quote.len() < CLOSE_GROUP_SIZE { retries += 1; error!("Error while fetching store quote: not enough quotes ({}/{CLOSE_GROUP_SIZE}), retry #{retries}, quotes {quote:?}", @@ -182,39 +248,6 @@ async fn fetch_store_quote_with_retries( } // Shall have a sleep between retries to avoid choking the network. // This shall be rare to happen though. - std::thread::sleep(std::time::Duration::from_secs(5)); - } -} - -async fn get_market_price_with_rate_limiter_and_retries( - evm_network: &EvmNetwork, - rate_limiter: &mut RateLimiter, - quoting_metrics: Vec, -) -> Result, ant_evm::payment_vault::error::Error> { - const MAX_RETRIES: u64 = 2; - let mut retries: u64 = 0; - let mut interval_in_ms: u64 = 1000; - - loop { - rate_limiter - .wait_interval_since_last_request(interval_in_ms) - .await; - - match get_market_price(evm_network, quoting_metrics.clone()).await { - Ok(amounts) => { - break Ok(amounts); - } - Err(err) => { - if err.to_string().contains("429") && retries < MAX_RETRIES { - retries += 1; - interval_in_ms *= retries * 2; - error!("Error while fetching quote market price: {err:?}, retry #{retries}"); - continue; - } else { - error!("Error while fetching quote market price: {err:?}, stopping after {retries} retries"); - break Err(err); - } - } - } + tokio::time::sleep(std::time::Duration::from_secs(5)).await; } } diff --git a/autonomi/src/client/rate_limiter.rs b/autonomi/src/client/rate_limiter.rs deleted file mode 100644 index b1935f6e83..0000000000 --- a/autonomi/src/client/rate_limiter.rs +++ /dev/null @@ -1,27 +0,0 @@ -use ant_networking::target_arch::{sleep, Duration, Instant}; - -pub struct RateLimiter { - last_request_time: Option, -} - -impl RateLimiter { - pub fn new() -> Self { - Self { - last_request_time: None, - } - } - - pub async fn wait_interval_since_last_request(&mut self, interval_in_ms: u64) { - if let Some(last_request_time) = self.last_request_time { - let elapsed_time = last_request_time.elapsed(); - - let interval = Duration::from_millis(interval_in_ms); - - if elapsed_time < interval { - sleep(interval - elapsed_time).await; - } - } - - self.last_request_time = Some(Instant::now()); - } -} diff --git a/autonomi/src/client/registers.rs b/autonomi/src/client/registers.rs deleted file mode 100644 index dc56e37b45..0000000000 --- a/autonomi/src/client/registers.rs +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -#![allow(deprecated)] - -use crate::client::data::PayError; -use crate::client::Client; -use crate::client::ClientEvent; -use crate::client::UploadSummary; - -pub use ant_registers::{Permissions as RegisterPermissions, RegisterAddress}; -pub use bls::SecretKey as RegisterSecretKey; - -use ant_evm::{Amount, AttoTokens, EvmWallet, EvmWalletError}; -use ant_networking::{GetRecordCfg, GetRecordError, NetworkError, PutRecordCfg, VerificationKind}; -use ant_protocol::{ - storage::{try_deserialize_record, try_serialize_record, RecordKind, RetryStrategy}, - NetworkAddress, -}; -use ant_registers::Register as BaseRegister; -use ant_registers::{Permissions, RegisterCrdt, RegisterOp, SignedRegister}; -use bytes::Bytes; -use libp2p::kad::{Quorum, Record}; -use std::collections::BTreeSet; -use xor_name::XorName; - -use super::data::CostError; - -#[derive(Debug, thiserror::Error)] -pub enum RegisterError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("Network error")] - Network(#[from] NetworkError), - #[error("Serialization error")] - Serialization, - #[error("Register could not be verified (corrupt)")] - FailedVerification, - #[error("Payment failure occurred during register creation.")] - Pay(#[from] PayError), - #[error("Failed to retrieve wallet payment")] - Wallet(#[from] EvmWalletError), - #[error("Failed to write to low-level register")] - Write(#[source] ant_registers::Error), - #[error("Failed to sign register")] - CouldNotSign(#[source] ant_registers::Error), - #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another register name")] - InvalidQuote, - #[error("The payment proof contains no payees.")] - PayeesMissing, -} - -#[deprecated( - since = "0.2.4", - note = "Use transactions instead (see Client::transaction_put)" -)] -#[derive(Clone, Debug)] -pub struct Register { - signed_reg: SignedRegister, - crdt_reg: RegisterCrdt, -} - -impl Register { - pub fn address(&self) -> &RegisterAddress { - self.signed_reg.address() - } - - /// Retrieve the current values of the register. There can be multiple values - /// in case a register was updated concurrently. This is because of the nature - /// of registers, which allows for network concurrency. - pub fn values(&self) -> Vec { - self.crdt_reg - .read() - .into_iter() - .map(|(_hash, value)| value.into()) - .collect() - } - - fn new( - initial_value: Option, - name: XorName, - owner: RegisterSecretKey, - permissions: RegisterPermissions, - ) -> Result { - let pk = owner.public_key(); - - let base_register = BaseRegister::new(pk, name, permissions); - - let signature = owner.sign(base_register.bytes().map_err(RegisterError::Write)?); - let signed_reg = SignedRegister::new(base_register, signature, BTreeSet::new()); - - let crdt_reg = RegisterCrdt::new(*signed_reg.address()); - - let mut register = Register { - signed_reg, - crdt_reg, - }; - - if let Some(value) = initial_value { - register.write_atop(&value, &owner)?; - } - debug!( - "Created register {:?} with address: {:?}", - register, - register.address() - ); - Ok(register) - } - - fn write_atop(&mut self, entry: &[u8], owner: &RegisterSecretKey) -> Result<(), RegisterError> { - let children: BTreeSet<_> = self.crdt_reg.read().iter().map(|(hash, _)| *hash).collect(); - - let (_hash, address, crdt_op) = self - .crdt_reg - .write(entry.to_vec(), &children) - .map_err(RegisterError::Write)?; - - let op = RegisterOp::new(address, crdt_op, owner); - - let _ = self.signed_reg.add_op(op); - - Ok(()) - } -} - -#[deprecated( - since = "0.2.4", - note = "Use transactions instead (see Client::transaction_put)" -)] -impl Client { - /// Generate a new register key - pub fn register_generate_key() -> RegisterSecretKey { - RegisterSecretKey::random() - } - - /// Fetches a Register from the network. - pub async fn register_get(&self, address: RegisterAddress) -> Result { - info!("Fetching register at addr: {address}"); - let network_address = NetworkAddress::from_register_address(address); - let key = network_address.to_record_key(); - - let get_cfg = GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: None, - target_record: None, - expected_holders: Default::default(), - is_register: true, - }; - - let signed_reg = match self.network.get_record_from_network(key, &get_cfg).await { - Ok(record) => { - let signed_reg: SignedRegister = - try_deserialize_record(&record).map_err(|_| RegisterError::Serialization)?; - signed_reg - } - Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { result_map })) => { - error!("Got split record error for register at address: {address}. This should've been handled at the network layer"); - Err(RegisterError::Network(NetworkError::GetRecordError( - GetRecordError::SplitRecord { result_map }, - )))? - } - Err(e) => { - error!("Failed to get register {address:?} from network: {e}"); - Err(e)? - } - }; - - // Make sure the fetched record contains valid CRDT operations - signed_reg - .verify() - .map_err(|_| RegisterError::FailedVerification)?; - - let mut crdt_reg = RegisterCrdt::new(*signed_reg.address()); - for op in signed_reg.ops() { - if let Err(err) = crdt_reg.apply_op(op.clone()) { - return Err(RegisterError::Write(err)); - } - } - - let register = Register { - signed_reg, - crdt_reg, - }; - debug!("Fetched register {register:?} from the address: {address} in the network"); - Ok(register) - } - - /// Updates a Register on the network with a new value. This will overwrite existing value(s). - pub async fn register_update( - &self, - mut register: Register, - new_value: Bytes, - owner: RegisterSecretKey, - ) -> Result<(), RegisterError> { - register.write_atop(&new_value, &owner)?; - - let signed_register = register.signed_reg.clone(); - - // Prepare the record for network storage - let record = Record { - key: NetworkAddress::from_register_address(*register.address()).to_record_key(), - value: try_serialize_record(&signed_register, RecordKind::Register) - .map_err(|_| RegisterError::Serialization)? - .to_vec(), - publisher: None, - expires: None, - }; - - let get_cfg = GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: Some(RetryStrategy::default()), - target_record: None, - expected_holders: Default::default(), - is_register: true, - }; - let put_cfg = PutRecordCfg { - put_quorum: Quorum::All, - retry_strategy: None, - use_put_record_to: None, - verification: Some((VerificationKind::Crdt, get_cfg)), - }; - - // Store the updated register on the network - self.network - .put_record(record, &put_cfg) - .await - .inspect_err(|err| { - error!( - "Failed to put record - register {:?} to the network: {err}", - register.address() - ) - })?; - debug!( - "Updated register {:?} with new value {:?}", - register.address(), - new_value - ); - Ok(()) - } - - /// Get the cost to create a register - pub async fn register_cost( - &self, - name: String, - owner: RegisterSecretKey, - ) -> Result { - trace!("Getting cost for register with name: {name}"); - // get register address - let pk = owner.public_key(); - let name = XorName::from_content_parts(&[name.as_bytes()]); - let permissions = Permissions::new_with([pk]); - let register = Register::new(None, name, owner, permissions)?; - let reg_xor = register.address().xorname(); - - // get cost to store register - // NB TODO: register should be priced differently from other data - let store_quote = self.get_store_quotes(std::iter::once(reg_xor)).await?; - - let total_cost = AttoTokens::from_atto( - store_quote - .0 - .values() - .map(|quote| quote.price()) - .sum::(), - ); - debug!("Calculated the cost to create register with name: {name} is {total_cost}"); - Ok(total_cost) - } - - /// Get the address of a register from its name and owner - pub fn register_address(name: &str, owner: &RegisterSecretKey) -> RegisterAddress { - let pk = owner.public_key(); - let name = XorName::from_content_parts(&[name.as_bytes()]); - RegisterAddress::new(name, pk) - } - - /// Creates a new Register with a name and optional initial value and uploads it to the network. - /// - /// The Register is created with the owner as the only writer. - pub async fn register_create( - &self, - value: Option, - name: &str, - owner: RegisterSecretKey, - wallet: &EvmWallet, - ) -> Result { - let pk = owner.public_key(); - let permissions = Permissions::new_with([pk]); - - self.register_create_with_permissions(value, name, owner, permissions, wallet) - .await - } - - /// Creates a new Register with a name and optional initial value and uploads it to the network. - /// - /// Unlike `register_create`, this function allows you to specify the permissions for the register. - pub async fn register_create_with_permissions( - &self, - value: Option, - name: &str, - owner: RegisterSecretKey, - permissions: RegisterPermissions, - wallet: &EvmWallet, - ) -> Result { - info!("Creating register with name: {name}"); - let name = XorName::from_content_parts(&[name.as_bytes()]); - - // Owner can write to the register. - let register = Register::new(value, name, owner, permissions)?; - let address = register.address(); - - let reg_xor = address.xorname(); - debug!("Paying for register at address: {address}"); - let payment_proofs = self - .pay(std::iter::once(reg_xor), wallet) - .await - .inspect_err(|err| { - error!("Failed to pay for register at address: {address} : {err}") - })?; - let (proof, price) = if let Some((proof, price)) = payment_proofs.get(®_xor) { - (proof, price) - } else { - // register was skipped, meaning it was already paid for - error!("Register at address: {address} was already paid for"); - return Err(RegisterError::Network(NetworkError::RegisterAlreadyExists)); - }; - - let payees = proof.payees(); - let signed_register = register.signed_reg.clone(); - - let record = Record { - key: NetworkAddress::from_register_address(*address).to_record_key(), - value: try_serialize_record( - &(proof, &signed_register), - RecordKind::RegisterWithPayment, - ) - .map_err(|_| RegisterError::Serialization)? - .to_vec(), - publisher: None, - expires: None, - }; - - let get_cfg = GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: Some(RetryStrategy::default()), - target_record: None, - expected_holders: Default::default(), - is_register: true, - }; - - let put_cfg = PutRecordCfg { - put_quorum: Quorum::All, - retry_strategy: None, - use_put_record_to: Some(payees), - verification: Some((VerificationKind::Crdt, get_cfg)), - }; - - debug!("Storing register at address {address} to the network"); - self.network - .put_record(record, &put_cfg) - .await - .inspect_err(|err| { - error!("Failed to put record - register {address} to the network: {err}") - })?; - - if let Some(channel) = self.client_event_sender.as_ref() { - let summary = UploadSummary { - record_count: 1, - tokens_spent: price.as_atto(), - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err}"); - } - } - - Ok(register) - } -} diff --git a/autonomi/src/client/transactions.rs b/autonomi/src/client/transactions.rs deleted file mode 100644 index 1585709960..0000000000 --- a/autonomi/src/client/transactions.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::client::data::PayError; -use crate::client::Client; -use crate::client::ClientEvent; -use crate::client::UploadSummary; - -use ant_evm::Amount; -use ant_evm::AttoTokens; -pub use ant_protocol::storage::Transaction; -use ant_protocol::storage::TransactionAddress; -pub use bls::SecretKey; - -use ant_evm::{EvmWallet, EvmWalletError}; -use ant_networking::{GetRecordCfg, NetworkError, PutRecordCfg, VerificationKind}; -use ant_protocol::{ - storage::{try_serialize_record, RecordKind, RetryStrategy}, - NetworkAddress, -}; -use libp2p::kad::{Quorum, Record}; - -use super::data::CostError; - -#[derive(Debug, thiserror::Error)] -pub enum TransactionError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("Network error")] - Network(#[from] NetworkError), - #[error("Serialization error")] - Serialization, - #[error("Transaction could not be verified (corrupt)")] - FailedVerification, - #[error("Payment failure occurred during transaction creation.")] - Pay(#[from] PayError), - #[error("Failed to retrieve wallet payment")] - Wallet(#[from] EvmWalletError), - #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another transaction name")] - InvalidQuote, - #[error("Transaction already exists at this address: {0:?}")] - TransactionAlreadyExists(TransactionAddress), -} - -impl Client { - /// Fetches a Transaction from the network. - pub async fn transaction_get( - &self, - address: TransactionAddress, - ) -> Result, TransactionError> { - let transactions = self.network.get_transactions(address).await?; - - Ok(transactions) - } - - pub async fn transaction_put( - &self, - transaction: Transaction, - wallet: &EvmWallet, - ) -> Result<(), TransactionError> { - let address = transaction.address(); - - // pay for the transaction - let xor_name = address.xorname(); - debug!("Paying for transaction at address: {address:?}"); - let payment_proofs = self - .pay(std::iter::once(*xor_name), wallet) - .await - .inspect_err(|err| { - error!("Failed to pay for transaction at address: {address:?} : {err}") - })?; - - // make sure the transaction was paid for - let (proof, price) = match payment_proofs.get(xor_name) { - Some((proof, price)) => (proof, price), - None => { - // transaction was skipped, meaning it was already paid for - error!("Transaction at address: {address:?} was already paid for"); - return Err(TransactionError::TransactionAlreadyExists(address)); - } - }; - - // prepare the record for network storage - let payees = proof.payees(); - let record = Record { - key: NetworkAddress::from_transaction_address(address).to_record_key(), - value: try_serialize_record(&(proof, &transaction), RecordKind::TransactionWithPayment) - .map_err(|_| TransactionError::Serialization)? - .to_vec(), - publisher: None, - expires: None, - }; - let get_cfg = GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: Some(RetryStrategy::default()), - target_record: None, - expected_holders: Default::default(), - is_register: false, - }; - let put_cfg = PutRecordCfg { - put_quorum: Quorum::All, - retry_strategy: None, - use_put_record_to: Some(payees), - verification: Some((VerificationKind::Crdt, get_cfg)), - }; - - // put the record to the network - debug!("Storing transaction at address {address:?} to the network"); - self.network - .put_record(record, &put_cfg) - .await - .inspect_err(|err| { - error!("Failed to put record - transaction {address:?} to the network: {err}") - })?; - - // send client event - if let Some(channel) = self.client_event_sender.as_ref() { - let summary = UploadSummary { - record_count: 1, - tokens_spent: price.as_atto(), - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err}"); - } - } - - Ok(()) - } - - /// Get the cost to create a transaction - pub async fn transaction_cost(&self, key: SecretKey) -> Result { - let pk = key.public_key(); - trace!("Getting cost for transaction of {pk:?}"); - - let address = TransactionAddress::from_owner(pk); - let xor = *address.xorname(); - let store_quote = self.get_store_quotes(std::iter::once(xor)).await?; - let total_cost = AttoTokens::from_atto( - store_quote - .0 - .values() - .map(|quote| quote.price()) - .sum::(), - ); - debug!("Calculated the cost to create transaction of {pk:?} is {total_cost}"); - Ok(total_cost) - } -} diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index ad2aeececb..af123dca2a 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -6,197 +6,8 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::payment::{receipt_from_store_quotes, Receipt}; -use ant_evm::{EvmWallet, ProofOfPayment}; -use ant_networking::{GetRecordCfg, PutRecordCfg, VerificationKind}; -use ant_protocol::{ - messages::ChunkProof, - storage::{try_serialize_record, Chunk, RecordKind, RetryStrategy}, -}; -use bytes::Bytes; use futures::stream::{FuturesUnordered, StreamExt}; -use libp2p::kad::{Quorum, Record}; -use rand::{thread_rng, Rng}; -use self_encryption::{decrypt_full_set, DataMap, EncryptedChunk}; -use std::{future::Future, num::NonZero}; -use xor_name::XorName; - -use super::{ - data::{GetError, PayError, PutError, CHUNK_DOWNLOAD_BATCH_SIZE}, - Client, -}; -use crate::self_encryption::DataMapLevel; - -impl Client { - /// Fetch and decrypt all chunks in the data map. - pub(crate) async fn fetch_from_data_map(&self, data_map: &DataMap) -> Result { - debug!("Fetching encrypted data chunks from data map {data_map:?}"); - let mut download_tasks = vec![]; - for info in data_map.infos() { - download_tasks.push(async move { - match self - .chunk_get(info.dst_hash) - .await - .inspect_err(|err| error!("Error fetching chunk {:?}: {err:?}", info.dst_hash)) - { - Ok(chunk) => Ok(EncryptedChunk { - index: info.index, - content: chunk.value, - }), - Err(err) => { - error!("Error fetching chunk {:?}: {err:?}", info.dst_hash); - Err(err) - } - } - }); - } - debug!("Successfully fetched all the encrypted chunks"); - let encrypted_chunks = - process_tasks_with_max_concurrency(download_tasks, *CHUNK_DOWNLOAD_BATCH_SIZE) - .await - .into_iter() - .collect::, GetError>>()?; - - let data = decrypt_full_set(data_map, &encrypted_chunks).map_err(|e| { - error!("Error decrypting encrypted_chunks: {e:?}"); - GetError::Decryption(crate::self_encryption::Error::SelfEncryption(e)) - })?; - debug!("Successfully decrypted all the chunks"); - Ok(data) - } - - /// Unpack a wrapped data map and fetch all bytes using self-encryption. - pub(crate) async fn fetch_from_data_map_chunk( - &self, - data_map_bytes: &Bytes, - ) -> Result { - let mut data_map_level: DataMapLevel = rmp_serde::from_slice(data_map_bytes) - .map_err(GetError::InvalidDataMap) - .inspect_err(|err| error!("Error deserializing data map: {err:?}"))?; - - loop { - let data_map = match &data_map_level { - DataMapLevel::First(map) => map, - DataMapLevel::Additional(map) => map, - }; - - let data = self.fetch_from_data_map(data_map).await?; - - match &data_map_level { - DataMapLevel::First(_) => break Ok(data), - DataMapLevel::Additional(_) => { - data_map_level = rmp_serde::from_slice(&data).map_err(|err| { - error!("Error deserializing data map: {err:?}"); - GetError::InvalidDataMap(err) - })?; - continue; - } - }; - } - } - - pub(crate) async fn chunk_upload_with_payment( - &self, - chunk: &Chunk, - payment: ProofOfPayment, - ) -> Result<(), PutError> { - let storing_nodes = payment.payees(); - - if storing_nodes.is_empty() { - return Err(PutError::PayeesMissing); - } - - debug!("Storing chunk: {chunk:?} to {:?}", storing_nodes); - - let key = chunk.network_address().to_record_key(); - - let record_kind = RecordKind::ChunkWithPayment; - let record = Record { - key: key.clone(), - value: try_serialize_record(&(payment, chunk.clone()), record_kind) - .map_err(|e| { - PutError::Serialization(format!( - "Failed to serialize chunk with payment: {e:?}" - )) - })? - .to_vec(), - publisher: None, - expires: None, - }; - - let verification = { - let verification_cfg = GetRecordCfg { - get_quorum: Quorum::N(NonZero::new(2).expect("2 is non-zero")), - retry_strategy: Some(RetryStrategy::Balanced), - target_record: None, - expected_holders: Default::default(), - is_register: false, - }; - - let stored_on_node = try_serialize_record(&chunk, RecordKind::Chunk) - .map_err(|e| PutError::Serialization(format!("Failed to serialize chunk: {e:?}")))? - .to_vec(); - let random_nonce = thread_rng().gen::(); - let expected_proof = ChunkProof::new(&stored_on_node, random_nonce); - - Some(( - VerificationKind::ChunkProof { - expected_proof, - nonce: random_nonce, - }, - verification_cfg, - )) - }; - - let put_cfg = PutRecordCfg { - put_quorum: Quorum::One, - retry_strategy: Some(RetryStrategy::Balanced), - use_put_record_to: Some(storing_nodes.clone()), - verification, - }; - let payment_upload = Ok(self.network.put_record(record, &put_cfg).await?); - debug!("Successfully stored chunk: {chunk:?} to {storing_nodes:?}"); - payment_upload - } - - /// Pay for the chunks and get the proof of payment. - pub(crate) async fn pay( - &self, - content_addrs: impl Iterator + Clone, - wallet: &EvmWallet, - ) -> Result { - let number_of_content_addrs = content_addrs.clone().count(); - let quotes = self.get_store_quotes(content_addrs).await?; - - // Make sure nobody else can use the wallet while we are paying - debug!("Waiting for wallet lock"); - let lock_guard = wallet.lock().await; - debug!("Locked wallet"); - - // TODO: the error might contain some succeeded quote payments as well. These should be returned on err, so that they can be skipped when retrying. - // TODO: retry when it fails? - // Execute chunk payments - let _payments = wallet - .pay_for_quotes(quotes.payments()) - .await - .map_err(|err| PayError::from(err.0))?; - - // payment is done, unlock the wallet for other threads - drop(lock_guard); - debug!("Unlocked wallet"); - - let skipped_chunks = number_of_content_addrs - quotes.len(); - trace!( - "Chunk payments of {} chunks completed. {} chunks were free / already paid for", - quotes.len(), - skipped_chunks - ); - - let receipt = receipt_from_store_quotes(quotes); - - Ok(receipt) - } -} +use std::future::Future; pub(crate) async fn process_tasks_with_max_concurrency(tasks: I, batch_size: usize) -> Vec where diff --git a/autonomi/src/client/vault.rs b/autonomi/src/client/vault.rs deleted file mode 100644 index dd69f8f9d7..0000000000 --- a/autonomi/src/client/vault.rs +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -pub mod key; -pub mod user_data; - -pub use key::{derive_vault_key, VaultSecretKey}; -pub use user_data::UserData; - -use super::data::CostError; -use crate::client::data::PutError; -use crate::client::payment::PaymentOption; -use crate::client::Client; -use ant_evm::{Amount, AttoTokens}; -use ant_networking::{GetRecordCfg, GetRecordError, NetworkError, PutRecordCfg, VerificationKind}; -use ant_protocol::storage::{ - try_serialize_record, RecordKind, RetryStrategy, Scratchpad, ScratchpadAddress, -}; -use ant_protocol::Bytes; -use ant_protocol::{storage::try_deserialize_record, NetworkAddress}; -use libp2p::kad::{Quorum, Record}; -use std::collections::HashSet; -use std::hash::{DefaultHasher, Hash, Hasher}; -use tracing::info; - -#[derive(Debug, thiserror::Error)] -pub enum VaultError { - #[error("Could not generate Vault secret key from entropy: {0:?}")] - Bls(#[from] bls::Error), - #[error("Scratchpad found at {0:?} was not a valid record.")] - CouldNotDeserializeVaultScratchPad(ScratchpadAddress), - #[error("Protocol: {0}")] - Protocol(#[from] ant_protocol::Error), - #[error("Network: {0}")] - Network(#[from] NetworkError), - #[error("Vault not found")] - Missing, -} - -/// The content type of the vault data -/// The number is used to determine the type of the contents of the bytes contained in a vault -/// Custom apps can use this to store their own custom types of data in vaults -/// It is recommended to use the hash of the app name or an unique identifier as the content type using [`app_name_to_vault_content_type`] -/// The value 0 is reserved for tests -pub type VaultContentType = u64; - -/// For custom apps using Scratchpad, this function converts an app identifier or name to a [`VaultContentType`] -pub fn app_name_to_vault_content_type(s: T) -> VaultContentType { - let mut hasher = DefaultHasher::new(); - s.hash(&mut hasher); - hasher.finish() -} - -impl Client { - /// Retrieves and returns a decrypted vault if one exists. - /// Returns the content type of the bytes in the vault - pub async fn fetch_and_decrypt_vault( - &self, - secret_key: &VaultSecretKey, - ) -> Result<(Bytes, VaultContentType), VaultError> { - info!("Fetching and decrypting vault..."); - let pad = self.get_vault_from_network(secret_key).await?; - - let data = pad.decrypt_data(secret_key)?; - debug!("vault data is successfully fetched and decrypted"); - Ok((data, pad.data_encoding())) - } - - /// Gets the vault Scratchpad from a provided client public key - async fn get_vault_from_network( - &self, - secret_key: &VaultSecretKey, - ) -> Result { - let client_pk = secret_key.public_key(); - - let scratch_address = ScratchpadAddress::new(client_pk); - let network_address = NetworkAddress::from_scratchpad_address(scratch_address); - info!("Fetching vault from network at {network_address:?}",); - let scratch_key = network_address.to_record_key(); - - let get_cfg = GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: None, - target_record: None, - expected_holders: HashSet::new(), - is_register: false, - }; - - let pad = match self - .network - .get_record_from_network(scratch_key.clone(), &get_cfg) - .await - { - Ok(record) => { - debug!("Got scratchpad for {scratch_key:?}"); - try_deserialize_record::(&record) - .map_err(|_| VaultError::CouldNotDeserializeVaultScratchPad(scratch_address))? - } - Err(NetworkError::GetRecordError(GetRecordError::SplitRecord { result_map })) => { - debug!("Got multiple scratchpads for {scratch_key:?}"); - let mut pads = result_map - .values() - .map(|(record, _)| try_deserialize_record::(record)) - .collect::, _>>() - .map_err(|_| VaultError::CouldNotDeserializeVaultScratchPad(scratch_address))?; - - // take the latest versions - pads.sort_by_key(|s| s.count()); - let max_version = pads.last().map(|p| p.count()).unwrap_or_else(|| { - error!("Got empty scratchpad vector for {scratch_key:?}"); - u64::MAX - }); - let latest_pads: Vec<_> = pads - .into_iter() - .filter(|s| s.count() == max_version) - .collect(); - - // make sure we only have one of latest version - let pad = match &latest_pads[..] { - [one] => one, - [multi, ..] => { - error!("Got multiple conflicting scratchpads for {scratch_key:?} with the latest version, returning the first one"); - multi - } - [] => { - error!("Got empty scratchpad vector for {scratch_key:?}"); - return Err(VaultError::Missing); - } - }; - pad.to_owned() - } - Err(e) => { - warn!("Failed to fetch vault {network_address:?} from network: {e}"); - return Err(e)?; - } - }; - - Ok(pad) - } - - /// Get the cost of creating a new vault - pub async fn vault_cost(&self, owner: &VaultSecretKey) -> Result { - info!("Getting cost for vault"); - let client_pk = owner.public_key(); - let content_type = Default::default(); - let scratch = Scratchpad::new(client_pk, content_type); - let vault_xor = scratch.network_address().as_xorname().unwrap_or_default(); - - // NB TODO: vault should be priced differently from other data - let store_quote = self.get_store_quotes(std::iter::once(vault_xor)).await?; - - let total_cost = AttoTokens::from_atto( - store_quote - .0 - .values() - .map(|quote| quote.price()) - .sum::(), - ); - - Ok(total_cost) - } - - /// Put data into the client's VaultPacket - /// - /// Pays for a new VaultPacket if none yet created for the client. - /// Provide the bytes to be written to the vault and the content type of those bytes. - /// It is recommended to use the hash of the app name or unique identifier as the content type. - pub async fn write_bytes_to_vault( - &self, - data: Bytes, - payment_option: PaymentOption, - secret_key: &VaultSecretKey, - content_type: VaultContentType, - ) -> Result { - let mut total_cost = AttoTokens::zero(); - - let (mut scratch, is_new) = self - .get_or_create_scratchpad(secret_key, content_type) - .await?; - - let _ = scratch.update_and_sign(data, secret_key); - debug_assert!(scratch.is_valid(), "Must be valid after being signed. This is a bug, please report it by opening an issue on our github"); - - let scratch_address = scratch.network_address(); - let scratch_key = scratch_address.to_record_key(); - - info!("Writing to vault at {scratch_address:?}",); - - let record = if is_new { - let receipt = self - .pay_for_content_addrs(scratch.to_xor_name_vec().into_iter(), payment_option) - .await - .inspect_err(|err| { - error!("Failed to pay for new vault at addr: {scratch_address:?} : {err}"); - })?; - - let (proof, price) = match receipt.values().next() { - Some(proof) => proof, - None => return Err(PutError::PaymentUnexpectedlyInvalid(scratch_address)), - }; - - total_cost = *price; - - Record { - key: scratch_key, - value: try_serialize_record(&(proof, scratch), RecordKind::ScratchpadWithPayment) - .map_err(|_| { - PutError::Serialization( - "Failed to serialize scratchpad with payment".to_string(), - ) - })? - .to_vec(), - publisher: None, - expires: None, - } - } else { - Record { - key: scratch_key, - value: try_serialize_record(&scratch, RecordKind::Scratchpad) - .map_err(|_| { - PutError::Serialization("Failed to serialize scratchpad".to_string()) - })? - .to_vec(), - publisher: None, - expires: None, - } - }; - - let put_cfg = PutRecordCfg { - put_quorum: Quorum::Majority, - retry_strategy: Some(RetryStrategy::Balanced), - use_put_record_to: None, - verification: Some(( - VerificationKind::Crdt, - GetRecordCfg { - get_quorum: Quorum::Majority, - retry_strategy: None, - target_record: None, - expected_holders: HashSet::new(), - is_register: false, - }, - )), - }; - - debug!("Put record - scratchpad at {scratch_address:?} to the network"); - self.network - .put_record(record, &put_cfg) - .await - .inspect_err(|err| { - error!( - "Failed to put scratchpad {scratch_address:?} to the network with err: {err:?}" - ) - })?; - - Ok(total_cost) - } - - /// Returns an existing scratchpad or creates a new one if it does not exist. - pub async fn get_or_create_scratchpad( - &self, - secret_key: &VaultSecretKey, - content_type: VaultContentType, - ) -> Result<(Scratchpad, bool), PutError> { - let client_pk = secret_key.public_key(); - - let pad_res = self.get_vault_from_network(secret_key).await; - let mut is_new = true; - - let scratch = if let Ok(existing_data) = pad_res { - info!("Scratchpad already exists, returning existing data"); - - info!( - "scratch already exists, is version {:?}", - existing_data.count() - ); - - is_new = false; - - if existing_data.owner() != &client_pk { - return Err(PutError::VaultBadOwner); - } - - existing_data - } else { - trace!("new scratchpad creation"); - Scratchpad::new(client_pk, content_type) - }; - - Ok((scratch, is_new)) - } -} diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs deleted file mode 100644 index ce49ba83d2..0000000000 --- a/autonomi/src/client/wasm.rs +++ /dev/null @@ -1,868 +0,0 @@ -use super::address::{addr_to_str, str_to_addr}; -#[cfg(feature = "vault")] -use super::vault::UserData; -use crate::client::data::DataMapChunk; -use crate::client::payment::Receipt; -use ant_protocol::storage::Chunk; -use libp2p::Multiaddr; -use wasm_bindgen::prelude::*; - -/// The `Client` object allows interaction with the network to store and retrieve data. -/// -/// To connect to the network, see {@link Client.connect}. -/// -/// # Example -/// -/// ```js -/// let client = await Client.connect(["/ip4/127.0.0.1/tcp/36075/ws/p2p/12D3KooWALb...BhDAfJY"]); -/// const dataAddr = await client.putData(new Uint8Array([0, 1, 2, 3]), wallet); -/// -/// const archive = new Archive(); -/// archive.addFile("foo", dataAddr, createMetadata(4)); -/// -/// const archiveAddr = await client.putArchive(archive, wallet); -/// const archiveFetched = await client.getArchive(archiveAddr); -/// ``` -#[wasm_bindgen(js_name = Client)] -pub struct JsClient(super::Client); - -#[wasm_bindgen] -pub struct AttoTokens(ant_evm::AttoTokens); -#[wasm_bindgen] -impl AttoTokens { - #[wasm_bindgen(js_name = toString)] - pub fn to_string(&self) -> String { - self.0.to_string() - } -} - -#[wasm_bindgen(js_name = Chunk)] -pub struct JsChunk(Chunk); - -#[wasm_bindgen(js_class = Chunk)] -impl JsChunk { - /// Returns the bytes. - #[wasm_bindgen] - pub fn bytes(&self) -> Vec { - self.0.value.to_vec() - } - - /// Returns the XOR name. - #[wasm_bindgen] - pub fn xor_name(&self) -> String { - self.0.address.xorname().to_string() - } -} - -#[wasm_bindgen(js_class = Client)] -impl JsClient { - /// Connect to the network via the given peers. - /// - /// # Example - /// - /// ```js - /// let client = await Client.connect(["/ip4/127.0.0.1/tcp/36075/ws/p2p/12D3KooWALb...BhDAfJY"]); - /// ``` - #[wasm_bindgen] - pub async fn connect(peers: Vec) -> Result { - let peers = peers - .into_iter() - .map(|peer| peer.parse()) - .collect::, _>>()?; - - let client = super::Client::init_with_peers(peers).await?; - - Ok(JsClient(client)) - } - - /// Upload a chunk to the network. - /// - /// Returns the hex encoded address of the chunk. - /// - /// This is not yet implemented. - #[wasm_bindgen(js_name = putChunk)] - pub async fn put_chunk(&self, _data: Vec, _wallet: &JsWallet) -> Result { - async { unimplemented!() }.await - } - - /// Fetch the chunk from the network. - #[wasm_bindgen(js_name = getChunk)] - pub async fn get_chunk(&self, addr: String) -> Result, JsError> { - let addr = str_to_addr(&addr)?; - let chunk = self.0.chunk_get(addr).await?; - - Ok(chunk.value().to_vec()) - } - - /// Upload data to the network. - /// - /// Returns the hex encoded address of the data. - #[wasm_bindgen(js_name = putData)] - pub async fn put_data(&self, data: Vec, wallet: &JsWallet) -> Result { - let data = crate::Bytes::from(data); - let xorname = self.0.data_put_public(data, (&wallet.0).into()).await?; - - Ok(addr_to_str(xorname)) - } - - /// Upload private data to the network. - /// - /// Returns the `DataMapChunk` chunk of the data. - #[wasm_bindgen(js_name = putPrivateData)] - pub async fn put_private_data( - &self, - data: Vec, - wallet: &JsWallet, - ) -> Result { - let data = crate::Bytes::from(data); - let private_data_access = self.0.data_put(data, (&wallet.0).into()).await?; - let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; - - Ok(js_value) - } - - /// Upload private data to the network. - /// Uses a `Receipt` as payment. - /// - /// Returns the `DataMapChunk` chunk of the data. - #[wasm_bindgen(js_name = putPrivateDataWithReceipt)] - pub async fn put_private_data_with_receipt( - &self, - data: Vec, - receipt: JsValue, - ) -> Result { - let data = crate::Bytes::from(data); - let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let private_data_access = self.0.data_put(data, receipt.into()).await?; - let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; - - Ok(js_value) - } - - /// Fetch the data from the network. - #[wasm_bindgen(js_name = getData)] - pub async fn get_data(&self, addr: String) -> Result, JsError> { - let addr = str_to_addr(&addr)?; - let data = self.0.data_get_public(addr).await?; - - Ok(data.to_vec()) - } - - /// Fetch the data from the network. - #[wasm_bindgen(js_name = getPrivateData)] - pub async fn get_private_data(&self, private_data_access: JsValue) -> Result, JsError> { - let private_data_access: DataMapChunk = - serde_wasm_bindgen::from_value(private_data_access)?; - let data = self.0.data_get(private_data_access).await?; - - Ok(data.to_vec()) - } - - /// Get the cost of uploading data to the network. - #[wasm_bindgen(js_name = getDataCost)] - pub async fn get_data_cost(&self, data: Vec) -> Result { - let data = crate::Bytes::from(data); - let cost = self.0.data_cost(data).await.map_err(JsError::from)?; - - Ok(AttoTokens(cost)) - } -} - -mod archive { - use super::*; - use crate::client::{ - address::str_to_addr, files::archive::Metadata, files::archive_public::PublicArchive, - }; - use std::path::PathBuf; - use wasm_bindgen::JsError; - - /// Structure mapping paths to data addresses. - #[wasm_bindgen(js_name = Archive)] - pub struct JsArchive(PublicArchive); - - /// Create new metadata with the current time as uploaded, created and modified. - /// - /// # Example - /// - /// ```js - /// const metadata = createMetadata(BigInt(3)); - /// const archive = new atnm.Archive(); - /// archive.addFile("foo", addr, metadata); - /// ``` - #[wasm_bindgen(js_name = createMetadata)] - pub fn create_metadata(size: u64) -> Result { - let metadata = Metadata::new_with_size(size); - Ok(serde_wasm_bindgen::to_value(&metadata)?) - } - - #[wasm_bindgen(js_class = Archive)] - impl JsArchive { - /// Create a new archive. - #[wasm_bindgen(constructor)] - pub fn new() -> Self { - Self(PublicArchive::new()) - } - - /// Add a new file to the archive. - #[wasm_bindgen(js_name = addFile)] - pub fn add_file( - &mut self, - path: String, - data_addr: String, - metadata: JsValue, - ) -> Result<(), JsError> { - let path = PathBuf::from(path); - let data_addr = str_to_addr(&data_addr)?; - let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; - self.0.add_file(path, data_addr, metadata); - - Ok(()) - } - - #[wasm_bindgen(js_name = renameFile)] - pub fn rename_file(&mut self, old_path: String, new_path: String) -> Result<(), JsError> { - let old_path = PathBuf::from(old_path); - let new_path = PathBuf::from(new_path); - self.0.rename_file(&old_path, &new_path)?; - - Ok(()) - } - - #[wasm_bindgen] - pub fn map(&self) -> Result { - let files = serde_wasm_bindgen::to_value(self.0.map())?; - Ok(files) - } - - /// Serialize to bytes. - #[wasm_bindgen(js_name = bytes)] - pub fn into_bytes(&self) -> Result, JsError> { - let root_serialized = rmp_serde::to_vec(&self.0)?; - Ok(root_serialized) - } - } - - #[wasm_bindgen(js_class = Client)] - impl JsClient { - /// Fetch an archive from the network. - #[wasm_bindgen(js_name = getArchive)] - pub async fn get_archive(&self, addr: String) -> Result { - let addr = str_to_addr(&addr)?; - let archive = self.0.archive_get_public(addr).await?; - let archive = JsArchive(archive); - - Ok(archive) - } - - /// Upload an archive to the network. - /// - /// Returns the hex encoded address of the archive. - #[wasm_bindgen(js_name = putArchive)] - pub async fn put_archive( - &self, - archive: &JsArchive, - wallet: &JsWallet, - ) -> Result { - let addr = self.0.archive_put_public(&archive.0, &wallet.0).await?; - - Ok(addr_to_str(addr)) - } - } -} - -mod archive_private { - use super::*; - use crate::client::data::DataMapChunk; - use crate::client::files::archive::{Metadata, PrivateArchive, PrivateArchiveAccess}; - use crate::client::payment::Receipt; - use std::path::PathBuf; - use wasm_bindgen::{JsError, JsValue}; - - /// Structure mapping paths to data addresses. - #[wasm_bindgen(js_name = PrivateArchive)] - pub struct JsPrivateArchive(PrivateArchive); - - #[wasm_bindgen(js_class = PrivateArchive)] - impl JsPrivateArchive { - /// Create a new private archive. - #[wasm_bindgen(constructor)] - pub fn new() -> Self { - Self(PrivateArchive::new()) - } - - /// Add a new file to the private archive. - #[wasm_bindgen(js_name = addFile)] - pub fn add_file( - &mut self, - path: String, - data_map: JsValue, - metadata: JsValue, - ) -> Result<(), JsError> { - let path = PathBuf::from(path); - let data_map: DataMapChunk = serde_wasm_bindgen::from_value(data_map)?; - let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; - self.0.add_file(path, data_map, metadata); - - Ok(()) - } - - #[wasm_bindgen] - pub fn map(&self) -> Result { - let files = serde_wasm_bindgen::to_value(self.0.map())?; - Ok(files) - } - - /// Serialize to bytes. - #[wasm_bindgen(js_name = bytes)] - pub fn into_bytes(&self) -> Result, JsError> { - let root_serialized = rmp_serde::to_vec(&self.0)?; - Ok(root_serialized) - } - } - - #[wasm_bindgen(js_class = Client)] - impl JsClient { - /// Fetch a private archive from the network. - #[wasm_bindgen(js_name = getPrivateArchive)] - pub async fn get_private_archive( - &self, - private_archive_access: JsValue, - ) -> Result { - let private_archive_access: PrivateArchiveAccess = - serde_wasm_bindgen::from_value(private_archive_access)?; - let archive = self.0.archive_get(private_archive_access).await?; - let archive = JsPrivateArchive(archive); - - Ok(archive) - } - - /// Upload a private archive to the network. - /// - /// Returns the `PrivateArchiveAccess` chunk of the archive. - #[wasm_bindgen(js_name = putPrivateArchive)] - pub async fn put_private_archive( - &self, - archive: &JsPrivateArchive, - wallet: &JsWallet, - ) -> Result { - let private_archive_access = self.0.archive_put(&archive.0, (&wallet.0).into()).await?; - - let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; - - Ok(js_value) - } - - /// Upload a private archive to the network. - /// Uses a `Receipt` as payment. - /// - /// Returns the `PrivateArchiveAccess` chunk of the archive. - #[wasm_bindgen(js_name = putPrivateArchiveWithReceipt)] - pub async fn put_private_archive_with_receipt( - &self, - archive: &JsPrivateArchive, - receipt: JsValue, - ) -> Result { - let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - - let private_archive_access = self.0.archive_put(&archive.0, receipt.into()).await?; - - let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; - - Ok(js_value) - } - } -} - -#[cfg(feature = "vault")] -mod vault { - use super::*; - use crate::client::address::addr_to_str; - use crate::client::files::archive::PrivateArchiveAccess; - use crate::client::payment::Receipt; - use crate::client::vault::key::blst_to_blsttc; - use crate::client::vault::key::derive_secret_key_from_seed; - use crate::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; - use crate::client::vault::VaultContentType; - use ant_protocol::storage::Scratchpad; - use wasm_bindgen::{JsError, JsValue}; - - /// Structure to keep track of uploaded archives, registers and other data. - #[wasm_bindgen(js_name = UserData)] - pub struct JsUserData(UserData); - - #[wasm_bindgen(js_class = UserData)] - impl JsUserData { - /// Create a new user data structure. - #[wasm_bindgen(constructor)] - pub fn new() -> Self { - Self(UserData::new()) - } - - /// Store an archive address in the user data with an optional name. - /// - /// # Example - /// - /// ```js - /// userData.addFileArchive(archiveAddr, "foo"); - /// ``` - #[wasm_bindgen(js_name = addFileArchive)] - pub fn add_file_archive( - &mut self, - archive: String, - name: Option, - ) -> Result<(), JsError> { - let archive = str_to_addr(&archive)?; - - let old_name = if let Some(ref name) = name { - self.0.add_file_archive_with_name(archive, name.clone()) - } else { - self.0.add_file_archive(archive) - }; - - if let Some(old_name) = old_name { - tracing::warn!( - "Changing name of archive `{archive}` from `{old_name:?}` to `{name:?}`" - ); - } - - Ok(()) - } - - /// Store a private archive data map in the user data with an optional name. - /// - /// # Example - /// - /// ```js - /// userData.addPrivateFileArchive(privateArchiveAccess, "foo"); - /// ``` - #[wasm_bindgen(js_name = addPrivateFileArchive)] - pub fn add_private_file_archive( - &mut self, - private_archive_access: JsValue, - name: Option, - ) -> Result<(), JsError> { - let private_archive_access: PrivateArchiveAccess = - serde_wasm_bindgen::from_value(private_archive_access)?; - - let old_name = if let Some(ref name) = name { - self.0 - .add_private_file_archive_with_name(private_archive_access, name.clone()) - } else { - self.0.add_private_file_archive(private_archive_access) - }; - - if let Some(old_name) = old_name { - tracing::warn!( - "Changing name of private archive from `{old_name:?}` to `{name:?}`" - ); - } - - Ok(()) - } - - #[wasm_bindgen(js_name = removeFileArchive)] - pub fn remove_file_archive(&mut self, archive: String) -> Result<(), JsError> { - let archive = str_to_addr(&archive)?; - self.0.remove_file_archive(archive); - - Ok(()) - } - - #[wasm_bindgen(js_name = removePrivateFileArchive)] - pub fn remove_private_file_archive( - &mut self, - private_archive_access: JsValue, - ) -> Result<(), JsError> { - let private_archive_access: PrivateArchiveAccess = - serde_wasm_bindgen::from_value(private_archive_access)?; - - self.0.remove_private_file_archive(private_archive_access); - - Ok(()) - } - - #[wasm_bindgen(js_name = fileArchives)] - pub fn file_archives(&self) -> Result { - let archives = serde_wasm_bindgen::to_value(&self.0.file_archives)?; - Ok(archives) - } - - #[wasm_bindgen(js_name = privateFileArchives)] - pub fn private_file_archives(&self) -> Result { - let archives = serde_wasm_bindgen::to_value(&self.0.private_file_archives)?; - Ok(archives) - } - } - - #[wasm_bindgen(js_name = Scratchpad)] - pub struct JsScratchpad(Scratchpad); - - #[wasm_bindgen(js_class = Scratchpad)] - impl JsScratchpad { - /// Returns a VEC with the XOR name. - #[wasm_bindgen(js_name = xorName)] - pub fn xor_name(&self) -> Option { - self.0 - .network_address() - .as_xorname() - .map(|xor_name| addr_to_str(xor_name)) - } - } - - #[wasm_bindgen(js_class = Client)] - impl JsClient { - /// Fetch the user data from the vault. - /// - /// # Example - /// - /// ```js - /// const secretKey = genSecretKey(); - /// const userData = await client.getUserDataFromVault(secretKey); - /// ``` - #[wasm_bindgen(js_name = getUserDataFromVault)] - pub async fn get_user_data_from_vault( - &self, - secret_key: &SecretKeyJs, - ) -> Result { - let user_data = self.0.get_user_data_from_vault(&secret_key.0).await?; - - Ok(JsUserData(user_data)) - } - - /// Put the user data to the vault. - /// - /// # Example - /// - /// ```js - /// const secretKey = genSecretKey(); - /// await client.putUserDataToVault(userData, wallet, secretKey); - /// ``` - #[wasm_bindgen(js_name = putUserDataToVault)] - pub async fn put_user_data_to_vault( - &self, - user_data: &JsUserData, - wallet: &JsWallet, - secret_key: &SecretKeyJs, - ) -> Result<(), JsError> { - self.0 - .put_user_data_to_vault(&secret_key.0, (&wallet.0).into(), user_data.0.clone()) - .await?; - - Ok(()) - } - - /// Put the user data to the vault. - /// - /// # Example - /// - /// ```js - /// const secretKey = genSecretKey(); - /// await client.putUserDataToVaultWithReceipt(userData, receipt, secretKey); - /// ``` - #[wasm_bindgen(js_name = putUserDataToVaultWithReceipt)] - pub async fn put_user_data_to_vault_with_receipt( - &self, - user_data: &JsUserData, - receipt: JsValue, - secret_key: &SecretKeyJs, - ) -> Result<(), JsError> { - let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - - self.0 - .put_user_data_to_vault(&secret_key.0, receipt.into(), user_data.0.clone()) - .await?; - - Ok(()) - } - - /// Returns an existing scratchpad or creates a new one if it does not exist. - #[wasm_bindgen(js_name = getOrCreateScratchpad)] - pub async fn get_or_create_scratchpad( - &self, - secret_key: &SecretKeyJs, - vault_content_type: JsValue, - ) -> Result { - let vault_content_type: VaultContentType = - serde_wasm_bindgen::from_value(vault_content_type)?; - - let result = self - .0 - .get_or_create_scratchpad(&secret_key.0, vault_content_type) - .await?; - - let js_value = serde_wasm_bindgen::to_value(&result)?; - - Ok(js_value) - } - - /// Returns an existing user data scratchpad or creates a new one if it does not exist. - #[wasm_bindgen(js_name = getOrCreateUserDataScratchpad)] - pub async fn get_or_create_user_data_scratchpad( - &self, - secret_key: &SecretKeyJs, - ) -> Result { - let vault_content_type = *USER_DATA_VAULT_CONTENT_IDENTIFIER; - - let (scratchpad, _is_new) = self - .0 - .get_or_create_scratchpad(&secret_key.0, vault_content_type) - .await?; - - let js_scratchpad = JsScratchpad(scratchpad); - - Ok(js_scratchpad) - } - } - - #[wasm_bindgen(js_name = vaultKeyFromSignature)] - pub fn vault_key_from_signature(signature: Vec) -> Result { - let blst_key = derive_secret_key_from_seed(&signature)?; - let vault_sk = blst_to_blsttc(&blst_key)?; - Ok(SecretKeyJs(vault_sk)) - } -} - -#[cfg(feature = "external-signer")] -mod external_signer { - use super::*; - use crate::client::address::str_to_addr; - use crate::client::external_signer::encrypt_data; - use crate::client::payment::Receipt; - use crate::receipt_from_quotes_and_payments; - use ant_evm::external_signer::{approve_to_spend_tokens_calldata, pay_for_quotes_calldata}; - use ant_evm::EvmNetwork; - use ant_evm::QuotePayment; - use ant_evm::{Amount, PaymentQuote}; - use ant_evm::{EvmAddress, QuoteHash, TxHash}; - use std::collections::{BTreeMap, HashMap}; - use wasm_bindgen::prelude::wasm_bindgen; - use wasm_bindgen::{JsError, JsValue}; - use xor_name::XorName; - - #[wasm_bindgen(js_class = Client)] - impl JsClient { - /// Get quotes for given chunk addresses. - /// - /// # Example - /// - /// ```js - /// const [quotes, quotePayments, free_chunks] = await client.getQuotes(chunkAddresses); - /// `` - #[wasm_bindgen(js_name = getQuotes)] - pub async fn get_quotes(&self, chunk_addresses: Vec) -> Result { - let mut xor_addresses: Vec = vec![]; - - for chunk_address_str in &chunk_addresses { - let xor_address = str_to_addr(chunk_address_str)?; - xor_addresses.push(xor_address); - } - - let result = self - .0 - .get_quotes_for_content_addresses(xor_addresses.into_iter()) - .await?; - - let js_value = serde_wasm_bindgen::to_value(&result)?; - - Ok(js_value) - } - - /// Upload data with a receipt. - /// - /// # Example - /// - /// ```js - /// const receipt = getReceiptFromQuotesAndPayments(quotes, payments); - /// const addr = await client.putDataWithReceipt(data, receipt); - /// ``` - #[wasm_bindgen(js_name = putDataWithReceipt)] - pub async fn put_data_with_receipt( - &self, - data: Vec, - receipt: JsValue, - ) -> Result { - let data = crate::Bytes::from(data); - let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let xorname = self.0.data_put_public(data, receipt.into()).await?; - Ok(addr_to_str(xorname)) - } - } - - /// Encrypt data. - /// - /// # Example - /// - /// ```js - /// const [dataMapChunk, dataChunks, dataMapChunkAddress, dataChunkAddresses] = client.encryptData(data); - /// `` - #[wasm_bindgen(js_name = encryptData)] - pub fn encrypt(data: Vec) -> Result { - let data = crate::Bytes::from(data); - let result = encrypt_data(data)?; - let map_xor_name = *result.0.address().xorname(); - let mut xor_names = vec![]; - - for chunk in &result.1 { - xor_names.push(*chunk.name()); - } - - let result = (result.0, result.1, map_xor_name, xor_names); - let js_value = serde_wasm_bindgen::to_value(&result)?; - - Ok(js_value) - } - - /// Get the calldata for paying for quotes. - /// - /// # Example - /// - /// ```js - /// const [quotes, quotePayments, free_chunks] = await client.getQuotes(data); - /// const callData = getPayForQuotesCalldata(evmNetwork, quotePayments); - /// ``` - #[wasm_bindgen(js_name = getPayForQuotesCalldata)] - pub fn get_pay_for_quotes_calldata( - network: JsValue, - payments: JsValue, - ) -> Result { - let network: EvmNetwork = serde_wasm_bindgen::from_value(network)?; - let payments: Vec = serde_wasm_bindgen::from_value(payments)?; - let calldata = pay_for_quotes_calldata(&network, payments.into_iter())?; - let js_value = serde_wasm_bindgen::to_value(&calldata)?; - Ok(js_value) - } - - /// Form approve to spend tokens calldata. - #[wasm_bindgen(js_name = getApproveToSpendTokensCalldata)] - pub fn get_approve_to_spend_tokens_calldata( - network: JsValue, - spender: JsValue, - amount: JsValue, - ) -> Result { - let network: EvmNetwork = serde_wasm_bindgen::from_value(network)?; - let spender: EvmAddress = serde_wasm_bindgen::from_value(spender)?; - let amount: Amount = serde_wasm_bindgen::from_value(amount)?; - let calldata = approve_to_spend_tokens_calldata(&network, spender, amount); - let js_value = serde_wasm_bindgen::to_value(&calldata)?; - Ok(js_value) - } - - /// Generate payment proof. - #[wasm_bindgen(js_name = getReceiptFromQuotesAndPayments)] - pub fn get_receipt_from_quotes_and_payments( - quotes: JsValue, - payments: JsValue, - ) -> Result { - let quotes: HashMap = serde_wasm_bindgen::from_value(quotes)?; - let payments: BTreeMap = serde_wasm_bindgen::from_value(payments)?; - let receipt = receipt_from_quotes_and_payments("es, &payments); - let js_value = serde_wasm_bindgen::to_value(&receipt)?; - Ok(js_value) - } -} - -#[wasm_bindgen(js_name = SecretKey)] -pub struct SecretKeyJs(bls::SecretKey); - -/// # Example -/// -/// ```js -/// const secretKey = genSecretKey(); -/// await client.putUserDataToVault(userData, wallet, secretKey); -/// const userDataFetched = await client.getUserDataFromVault(secretKey); -/// ``` -#[wasm_bindgen(js_name = genSecretKey)] -pub fn gen_secret_key() -> SecretKeyJs { - let secret_key = bls::SecretKey::random(); - SecretKeyJs(secret_key) -} - -/// Get the current `EvmNetwork` that was set using environment variables that were used during the build process of this library. -#[wasm_bindgen(js_name = getEvmNetwork)] -pub fn evm_network() -> Result { - let evm_network = evmlib::utils::get_evm_network_from_env()?; - let js_value = serde_wasm_bindgen::to_value(&evm_network)?; - Ok(js_value) -} - -/// Create an `EvmNetwork` with custom values. -/// -/// # Example -/// -/// ```js -/// const [quotes, quotePayments, free_chunks] = await client.getQuotes(data); -/// const evmNetwork = getEvmNetworkCustom("http://localhost:4343", "", ""); -/// const payForQuotesCalldata = getPayForQuotesCalldata(evmNetwork, quotePayments); -/// ``` -#[wasm_bindgen(js_name = getEvmNetworkCustom)] -pub fn evm_network_custom( - rpc_url: String, - payment_token_address: String, - data_payments_address: String, -) -> Result { - let evm_network = - evmlib::utils::get_evm_network(&rpc_url, &payment_token_address, &data_payments_address); - let js_value = serde_wasm_bindgen::to_value(&evm_network)?; - Ok(js_value) -} - -#[wasm_bindgen(js_name = Wallet)] -pub struct JsWallet(evmlib::wallet::Wallet); - -/// Get a funded wallet for testing. This either uses a default private key or the `EVM_PRIVATE_KEY` -/// environment variable that was used during the build process of this library. -#[wasm_bindgen(js_name = getFundedWallet)] -pub fn funded_wallet() -> JsWallet { - let network = evmlib::utils::get_evm_network_from_env() - .expect("Failed to get EVM network from environment variables"); - if matches!(network, evmlib::Network::ArbitrumOne) { - panic!("You're trying to use ArbitrumOne network. Use a custom network for testing."); - } - // Default deployer wallet of the testnet. - const DEFAULT_WALLET_PRIVATE_KEY: &str = - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; - - let private_key = std::env::var("SECRET_KEY").unwrap_or(DEFAULT_WALLET_PRIVATE_KEY.to_string()); - - let wallet = evmlib::wallet::Wallet::new_from_private_key(network, &private_key) - .expect("Invalid private key"); - - JsWallet(wallet) -} - -/// Get a funded wallet with a custom network. -#[wasm_bindgen(js_name = getFundedWalletWithCustomNetwork)] -pub fn funded_wallet_with_custom_network( - network: JsValue, - private_key: String, -) -> Result { - let network: evmlib::Network = serde_wasm_bindgen::from_value(network)?; - let wallet = evmlib::wallet::Wallet::new_from_private_key(network, &private_key)?; - Ok(JsWallet(wallet)) -} - -/// Enable tracing logging in the console. -/// -/// A level could be passed like `trace` or `warn`. Or set for a specific module/crate -/// with `ant-networking=trace,autonomi=info`. -/// -/// # Example -/// -/// ```js -/// logInit("ant-networking=warn,autonomi=trace"); -/// ``` -#[wasm_bindgen(js_name = logInit)] -pub fn log_init(directive: String) { - use tracing_subscriber::prelude::*; - - console_error_panic_hook::set_once(); - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_ansi(false) // Only partially supported across browsers - .without_time() // std::time is not available in browsers - .with_writer(tracing_web::MakeWebConsoleWriter::new()); // write events to the console - tracing_subscriber::registry() - .with(fmt_layer) - .with(tracing_subscriber::EnvFilter::new(directive)) - .init(); -} diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index 81ff866006..50a11d1460 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -22,12 +22,12 @@ //! let wallet = Wallet::new_from_private_key(Default::default(), key)?; //! //! // Put and fetch data. -//! let data_addr = client.data_put_public(Bytes::from("Hello, World"), (&wallet).into()).await?; -//! let _data_fetched = client.data_get_public(data_addr).await?; +//! let (cost, data_addr) = client.data_put_public(Bytes::from("Hello, World"), (&wallet).into()).await?; +//! let _data_fetched = client.data_get_public(&data_addr).await?; //! //! // Put and fetch directory from local file system. -//! let dir_addr = client.dir_and_archive_upload_public("files/to/upload".into(), &wallet).await?; -//! client.dir_download_public(dir_addr, "files/downloaded".into()).await?; +//! let (cost, dir_addr) = client.dir_and_archive_upload_public("files/to/upload".into(), &wallet).await?; +//! client.dir_download_public(&dir_addr, "files/downloaded".into()).await?; //! //! Ok(()) //! } @@ -35,25 +35,15 @@ //! //! # Data types //! -//! This API gives access to two fundamental types on the network: chunks and -//! registers. +//! This API gives access to two fundamental types on the network: Chunks and GraphEntry. //! //! When we upload data, it's split into chunks using self-encryption, yielding //! a 'data map' allowing us to reconstruct the data again. Any two people that //! upload the exact same data will get the same data map, as all chunks are //! content-addressed and self-encryption is deterministic. //! -//! Registers can keep small values pointing to data. This value can be updated -//! and the history is kept. Multiple values can exist side by side in case of -//! concurrency, but should converge to a single value eventually. -//! //! # Features //! -//! - `fs`: Up/download files and directories from filesystem -//! - `registers`: Operate on register datatype -//! - `vault`: Operate on Vault datatype -//! - `full`: All of above -//! - `local`: Discover local peers using mDNS. Useful for development. //! - `loud`: Print debug information to stdout // docs.rs generation will enable unstable `doc_cfg` feature @@ -63,12 +53,33 @@ extern crate tracing; pub mod client; -mod self_encryption; +pub mod self_encryption; + +/// Client Operation config types +pub use ant_networking::{ResponseQuorum, RetryStrategy}; + +// The Network data types +pub use client::data_types::chunk; +pub use client::data_types::graph; +pub use client::data_types::pointer; +pub use client::data_types::scratchpad; + +// The high-level data types +pub use client::data; +pub use client::files; +pub use client::register; +pub use client::vault; -pub use ant_evm::get_evm_network_from_env; +// Re-exports of the evm types +pub use ant_evm::utils::get_evm_network; pub use ant_evm::EvmNetwork as Network; pub use ant_evm::EvmWallet as Wallet; +pub use ant_evm::QuoteHash; pub use ant_evm::RewardsAddress; +pub use ant_evm::{Amount, AttoTokens}; + +// Re-exports of the bls types +pub use bls::{PublicKey, SecretKey, Signature}; #[doc(no_inline)] // Place this under 'Re-exports' in the docs. pub use bytes::Bytes; @@ -76,7 +87,24 @@ pub use bytes::Bytes; pub use libp2p::Multiaddr; #[doc(inline)] -pub use client::{files::archive::Metadata, files::archive::PrivateArchive, Client, ClientConfig}; +pub use client::{ + // Client Configs + config::ClientConfig, + config::ClientOperatingStrategy, + + // Native data types + data_types::chunk::Chunk, + data_types::chunk::ChunkAddress, + data_types::graph::GraphEntry, + data_types::graph::GraphEntryAddress, + data_types::pointer::Pointer, + data_types::pointer::PointerAddress, + data_types::scratchpad::Scratchpad, + data_types::scratchpad::ScratchpadAddress, + + // Client + Client, +}; #[cfg(feature = "extension-module")] mod python; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index 1f1c4d443b..f313ec7e3e 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -1,236 +1,1419 @@ -// TODO: Shall be removed once the python binding warnings resolved -#![allow(non_local_definitions)] - -use crate::client::{ - data::DataMapChunk, - files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, - payment::PaymentOption as RustPaymentOption, - vault::{UserData, VaultSecretKey}, - Client as RustClient, +use std::{path::PathBuf, str::FromStr, sync::Arc}; + +use crate::{ + client::{ + chunk::DataMapChunk, + payment::PaymentOption, + vault::{UserData, VaultSecretKey}, + }, + files::{Metadata, PrivateArchive, PublicArchive}, + register::{RegisterAddress, RegisterHistory}, + Client, ClientConfig, +}; +use crate::{Bytes, Network, Wallet}; +use ant_protocol::storage::{ + Chunk, ChunkAddress, GraphEntry, GraphEntryAddress, Pointer, PointerAddress, PointerTarget, + Scratchpad, ScratchpadAddress, }; -use crate::{Bytes, Network, Wallet as RustWallet}; -use pyo3::exceptions::PyValueError; +use bls::{PublicKey, SecretKey}; +use libp2p::Multiaddr; +use pyo3::exceptions::{PyConnectionError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use pyo3_async_runtimes::tokio::future_into_py; use xor_name::XorName; -#[pyclass(name = "Client")] -pub(crate) struct PyClient { - inner: RustClient, +/// Represents a client for the Autonomi network. +// Missing methods: +// - upload_chunks_with_retries +// - enable_client_events +// - evm_network +// - get_store_quotes +// - pointer_verify +// - scratchpad_verify +#[pyclass(name = "Client")] +pub(crate) struct PyClient { + inner: Client, +} + +#[pymethods] +impl PyClient { + /// Initialize the client with default configuration. + #[staticmethod] + fn init(py: Python) -> PyResult> { + future_into_py(py, async { + let inner = Client::init() + .await + .map_err(|e| PyConnectionError::new_err(format!("Failed to connect: {e}")))?; + Ok(PyClient { inner }) + }) + } + + /// Initialize a client that is configured to be local. + #[staticmethod] + fn init_local(py: Python) -> PyResult> { + future_into_py(py, async { + let inner = Client::init_local() + .await + .map_err(|e| PyConnectionError::new_err(format!("Failed to connect: {e}")))?; + Ok(PyClient { inner }) + }) + } + + /// Initialize a client that bootstraps from a list of peers. + /// + /// If any of the provided peers is a global address, the client will not be local. + #[staticmethod] + fn init_with_peers(py: Python, peers: Vec) -> PyResult> { + let peers: Vec = peers + .iter() + .map(|p| Multiaddr::from_str(p)) + .collect::>() + .map_err(|e| PyValueError::new_err(format!("Failed to parse peers: {e}")))?; + + future_into_py(py, async { + let inner = Client::init_with_peers(peers) + .await + .map_err(|e| PyConnectionError::new_err(format!("Failed to connect: {e}")))?; + Ok(PyClient { inner }) + }) + } + + /// Initialize the client with the given configuration. + #[staticmethod] + fn init_with_config(py: Python, config: PyClientConfig) -> PyResult> { + future_into_py(py, async { + let inner = Client::init_with_config(config.inner) + .await + .map_err(|e| PyConnectionError::new_err(format!("Failed to connect: {e}")))?; + Ok(PyClient { inner }) + }) + } + + /// Get the cost of storing a chunk on the network + fn chunk_cost<'a>(&self, py: Python<'a>, addr: PyChunkAddress) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .chunk_cost(&addr.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get chunk cost: {e}")))?; + Ok(cost.to_string()) + }) + } + + /// Get a chunk from the network. + fn chunk_get<'a>(&self, py: Python<'a>, addr: &PyChunkAddress) -> PyResult> { + let client = self.inner.clone(); + let addr = addr.inner; + + future_into_py(py, async move { + let chunk = client + .chunk_get(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get chunk: {e}")))?; + Ok(chunk.value.to_vec()) + }) + } + + /// Manually upload a chunk to the network. It is recommended to use the `data_put` method instead to upload data. + fn chunk_put<'a>( + &self, + py: Python<'a>, + data: Vec, + payment: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment.inner.clone(); + let chunk = Chunk::new(Bytes::from(data)); + + future_into_py(py, async move { + let (cost, addr) = client + .chunk_put(&chunk, payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put chunk: {e}")))?; + Ok((cost.to_string(), PyChunkAddress::from(addr))) + }) + } + + /// Fetches a GraphEntry from the network. + fn graph_entry_get<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + ) -> PyResult> { + let client = self.inner.clone(); + let addr = GraphEntryAddress(addr); + + future_into_py(py, async move { + let entry = client + .graph_entry_get(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get graph entry: {e}")))?; + Ok(PyGraphEntry { inner: entry }) + }) + } + + /// Check if a graph_entry exists on the network + fn graph_entry_check_existance<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + ) -> PyResult> { + let client = self.inner.clone(); + let addr = GraphEntryAddress(addr); + + future_into_py(py, async move { + let exists = client + .graph_entry_check_existance(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get graph entry: {e}")))?; + Ok(exists) + }) + } + + /// Manually puts a GraphEntry to the network. + fn graph_entry_put<'a>( + &self, + py: Python<'a>, + entry: PyGraphEntry, + payment_option: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment_option.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .graph_entry_put(entry.inner, payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get graph entry: {e}")))?; + + Ok(( + cost.to_string(), + crate::client::address::addr_to_str(addr.0), + )) + }) + } + + /// Get the cost to create a GraphEntry + fn graph_entry_cost<'a>(&self, py: Python<'a>, key: PyPublicKey) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client.graph_entry_cost(&key.inner).await.map_err(|e| { + PyRuntimeError::new_err(format!("Failed to get graph entry cost: {e}")) + })?; + + Ok(cost.to_string()) + }) + } + + /// Get Scratchpad from the Network. + /// A Scratchpad is stored at the owner's public key so we can derive the address from it. + fn scratchpad_get_from_public_key<'a>( + &self, + py: Python<'a>, + public_key: PyPublicKey, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let scratchpad = client + .scratchpad_get_from_public_key(&public_key.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get scratchpad: {e}")))?; + + Ok(PyScratchpad { inner: scratchpad }) + }) + } + + /// Get Scratchpad from the Network using the scratpad address in hex string format. + fn scratchpad_get<'a>(&self, py: Python<'a>, addr: String) -> PyResult> { + let client = self.inner.clone(); + let addr = ScratchpadAddress::from_hex(&addr) + .map_err(|e| PyValueError::new_err(format!("Failed to parse address: {e}")))?; + + future_into_py(py, async move { + let scratchpad = client + .scratchpad_get(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get scratchpad: {e}")))?; + + Ok(PyScratchpad { inner: scratchpad }) + }) + } + + /// Check if a scratchpad exists on the network + fn scratchpad_check_existance<'a>( + &self, + py: Python<'a>, + addr: String, + ) -> PyResult> { + let client = self.inner.clone(); + let addr = ScratchpadAddress::from_hex(&addr) + .map_err(|e| PyValueError::new_err(format!("Failed to parse address: {e}")))?; + + future_into_py(py, async move { + let exists = client + .scratchpad_check_existance(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get scratchpad: {e}")))?; + + Ok(exists) + }) + } + + /// Manually store a scratchpad on the network + fn scratchpad_put<'a>( + &self, + py: Python<'a>, + scratchpad: PyScratchpad, + payment_option: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment_option.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .scratchpad_put(scratchpad.inner, payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put scratchpad: {e}")))?; + + Ok((cost.to_string(), addr.to_hex())) + }) + } + + /// Create a new scratchpad to the network. + /// + /// Make sure that the owner key is not already used for another scratchpad as each key is associated with one scratchpad. + /// The data will be encrypted with the owner key before being stored on the network. + /// The content type is used to identify the type of data stored in the scratchpad, the choice is up to the caller. + /// + /// Returns the cost and the address of the scratchpad. + fn scratchpad_create<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + content_type: u64, + initial_data: Vec, + payment_option: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment_option.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .scratchpad_create( + &owner.inner, + content_type, + &Bytes::from(initial_data), + payment, + ) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to create scratchpad: {e}")) + })?; + + Ok((cost.to_string(), addr.to_hex())) + }) + } + + /// Update an existing scratchpad to the network. + /// The scratchpad needs to be created first with `scratchpad_create`. + /// This operation is free as the scratchpad was already paid for at creation. + /// Only the latest version of the scratchpad is kept on the Network, previous versions will be overwritten and unrecoverable. + fn scratchpad_update<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + content_type: u64, + data: Vec, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .scratchpad_update(&owner.inner, content_type, &Bytes::from(data)) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to update scratchpad: {e}")) + })?; + + Ok(()) + }) + } + + /// Get the cost of creating a new Scratchpad + fn scratchpad_cost<'a>( + &self, + py: Python<'a>, + public_key: PyPublicKey, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .scratchpad_cost(&public_key.inner) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to get scratchpad cost: {e}")) + })?; + + Ok(cost.to_string()) + }) + } + + /// Get the cost of storing an archive on the network + fn archive_cost<'a>( + &self, + py: Python<'a>, + archive: PyPublicArchive, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .archive_cost(&archive.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get archive cost: {e}")))?; + Ok(cost.to_string()) + }) + } + + /// Fetch a private archive from the network using its data map + fn archive_get<'a>( + &self, + py: Python<'a>, + data_map: &PyDataMapChunk, + ) -> PyResult> { + let client = self.inner.clone(); + let data_map = data_map.inner.clone(); + + future_into_py(py, async move { + let archive = client + .archive_get(&data_map) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get archive: {e}")))?; + + Ok(PyPrivateArchive { inner: archive }) + }) + } + + /// Upload a private archive to the network + fn archive_put<'a>( + &self, + py: Python<'a>, + archive: PyPrivateArchive, + payment: PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, data_map) = client + .archive_put(&archive.inner, payment.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put archive: {e}")))?; + + Ok((cost.to_string(), PyDataMapChunk { inner: data_map })) + }) + } + + /// Upload a public archive to the network + fn archive_put_public<'a>( + &self, + py: Python<'a>, + archive: PyPublicArchive, + wallet: PyWallet, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .archive_put_public(&archive.inner, &wallet.inner) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to put public archive: {e}")) + })?; + + Ok((cost.to_string(), crate::client::address::addr_to_str(addr))) + }) + } + + /// Get the cost to upload a file/dir to the network. + fn file_cost<'a>(&self, py: Python<'a>, path: PathBuf) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .file_cost(&path) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get file cost: {e}")))?; + + Ok(cost.to_string()) + }) + } + + /// Download a private file from network to local file system. + fn file_download<'a>( + &self, + py: Python<'a>, + data_map: PyDataMapChunk, + path: PathBuf, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .file_download(&data_map.inner, path) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to download file: {e}")))?; + + Ok(()) + }) + } + + /// Download a private directory from network to local file system + fn dir_download<'a>( + &self, + py: Python<'a>, + data_map: PyDataMapChunk, + dir_path: PathBuf, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .dir_download(&data_map.inner, dir_path) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to download directory: {e}")) + })?; + Ok(()) + }) + } + + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// The data maps of these (private) files are not uploaded but returned within the PrivateArchive return type. + fn dir_upload<'a>( + &self, + py: Python<'a>, + dir_path: PathBuf, + wallet: PyWallet, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, archive) = client + .dir_upload(dir_path, &wallet.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to upload directory: {e}")))?; + Ok((cost.to_string(), PyPrivateArchive { inner: archive })) + }) + } + + /// Download file from network to local file system. + fn file_download_public<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + path: PathBuf, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .file_download_public(&addr, path) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to download public file: {e}")) + })?; + + Ok(()) + }) + } + + /// Same as `dir_upload` but also uploads the archive (privately) to the network. + /// + /// Returns the data map allowing the private archive to be downloaded from the network. + fn dir_and_archive_upload<'a>( + &self, + py: Python<'a>, + dir_path: PathBuf, + wallet: PyWallet, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, data_map) = client + .dir_and_archive_upload(dir_path, &wallet.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to upload directory: {e}")))?; + Ok((cost.to_string(), PyDataMapChunk { inner: data_map })) + }) + } + + /// Upload a piece of private data to the network. This data will be self-encrypted. + /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. + /// + /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. + fn data_put<'a>( + &self, + py: Python<'a>, + data: Vec, + payment: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment.inner.clone(); + + future_into_py(py, async move { + let (cost, data_map) = client + .data_put(Bytes::from(data), payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put data: {e}")))?; + Ok((cost.to_string(), PyDataMapChunk { inner: data_map })) + }) + } + + /// Fetch a blob of (private) data from the network + fn data_get<'a>(&self, py: Python<'a>, access: &PyDataMapChunk) -> PyResult> { + let client = self.inner.clone(); + let access = access.inner.clone(); + + future_into_py(py, async move { + let data = client + .data_get(&access) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get data: {e}")))?; + Ok(data.to_vec()) + }) + } + + /// Get the estimated cost of storing a piece of data. + fn data_cost<'a>(&self, py: Python<'a>, data: Vec) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .data_cost(Bytes::from(data)) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get data cost: {e}")))?; + Ok(cost.to_string()) + }) + } + + /// Upload a piece of data to the network. This data is publicly accessible. + /// + /// Returns the Data Address at which the data was stored. + fn data_put_public<'a>( + &self, + py: Python<'a>, + data: Vec, + payment: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .data_put_public(bytes::Bytes::from(data), payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put data: {e}")))?; + + Ok((cost.to_string(), crate::client::address::addr_to_str(addr))) + }) + } + + /// Fetch a blob of data from the network + fn data_get_public<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let data = client + .data_get_public(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get data: {e}")))?; + Ok(data.to_vec()) + }) + } + + /// Upload a directory as a public archive to the network. + /// Returns the network address where the archive is stored. + fn dir_and_archive_upload_public<'a>( + &self, + py: Python<'a>, + dir_path: PathBuf, + wallet: &PyWallet, + ) -> PyResult> { + let client = self.inner.clone(); + let wallet = wallet.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .dir_and_archive_upload_public(dir_path, &wallet) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to upload directory: {e}")))?; + Ok((cost.to_string(), crate::client::address::addr_to_str(addr))) + }) + } + + /// Download a public archive from the network to a local directory. + fn dir_download_public<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + dir_path: PathBuf, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .dir_download_public(&addr, dir_path) + .await + .map_err(|e| { + PyRuntimeError::new_err(format!("Failed to download directory: {e}")) + })?; + Ok(()) + }) + } + + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// + /// The data maps of these files are uploaded on the network, making the individual files publicly available. + /// + /// This returns, but does not upload (!),the `PublicArchive` containing the data maps of the uploaded files. + fn dir_upload_public<'a>( + &self, + py: Python<'a>, + dir_path: PathBuf, + wallet: PyWallet, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, archive) = client + .dir_upload_public(dir_path, &wallet.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to upload directory: {e}")))?; + Ok((cost.to_string(), PyPublicArchive { inner: archive })) + }) + } + + /// Get a public archive from the network. + fn archive_get_public<'a>( + &self, + py: Python<'a>, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let archive = client + .archive_get_public(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get archive: {e}")))?; + + Ok(PyPublicArchive { inner: archive }) + }) + } + + /// Get the cost of creating a new vault. + fn vault_cost<'a>( + &self, + py: Python<'a>, + key: &PyVaultSecretKey, + max_expected_size: u64, + ) -> PyResult> { + let client = self.inner.clone(); + let key = key.inner.clone(); + + future_into_py(py, async move { + let cost = client + .vault_cost(&key, max_expected_size) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get vault cost: {e}")))?; + Ok(cost.to_string()) + }) + } + + /// Put data into the client's VaultPacket + /// + /// Dynamically expand the vault capacity by paying for more space (Scratchpad) when needed. + /// + /// It is recommended to use the hash of the app name or unique identifier as the content type. + fn write_bytes_to_vault<'a>( + &self, + py: Python<'a>, + data: Vec, + payment: &PyPaymentOption, + key: &PyVaultSecretKey, + content_type: u64, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment.inner.clone(); + let key = key.inner.clone(); + + future_into_py(py, async move { + match client + .write_bytes_to_vault(bytes::Bytes::from(data), payment, &key, content_type) + .await + { + Ok(cost) => Ok(cost.to_string()), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to write to vault: {e}" + ))), + } + }) + } + + /// Get the register history, starting from the root to the latest entry. + /// + /// This returns a [`RegisterHistory`] that can be use to get the register values from the history. + /// + /// [`RegisterHistory::next`] can be used to get the values one by one, from the first to the latest entry. + /// [`RegisterHistory::collect`] can be used to get all the register values from the history from the first to the latest entry. + fn register_history(&self, addr: String) -> PyResult { + let client = self.inner.clone(); + let addr = RegisterAddress::from_hex(&addr) + .map_err(|e| PyValueError::new_err(format!("Failed to parse address: {e}")))?; + + let history = client.register_history(&addr); + Ok(PyRegisterHistory::new(history)) + } + + /// Create a new register key from a SecretKey and a name. + /// + /// This derives a new `SecretKey` from the owner's `SecretKey` using the name. + /// Note that you will need to keep track of the names you used to create the register key. + #[staticmethod] + fn register_key_from_name(owner: PySecretKey, name: &str) -> PyResult { + let key = Client::register_key_from_name(&owner.inner, name); + Ok(PySecretKey { inner: key }) + } + + /// Create a new RegisterValue from bytes, make sure the bytes are not longer than `REGISTER_VALUE_SIZE` + #[staticmethod] + fn register_value_from_bytes(bytes: &[u8]) -> PyResult<[u8; 32]> { + let value = Client::register_value_from_bytes(bytes) + .map_err(|e| PyValueError::new_err(format!("`bytes` has invalid length: {e}")))?; + Ok(value) + } + + /// Create a new register with an initial value. + /// + /// Note that two payments are required, one for the underlying `GraphEntry` and one for the `Pointer`. + fn register_create<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + value: [u8; 32], + payment: PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .register_create(&owner.inner, value, payment.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to create register: {e}")))?; + + Ok((cost.to_string(), addr.to_hex())) + }) + } + + /// Update the value of a register. + /// + /// The register needs to be created first with `register_create`. + fn register_update<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + value: [u8; 32], + payment: PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client + .register_update(&owner.inner, value, payment.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to update register: {e}")))?; + + Ok(cost.to_string()) + }) + } + + /// Get the current value of the register + fn register_get<'a>(&self, py: Python<'a>, addr: String) -> PyResult> { + let client = self.inner.clone(); + let addr = RegisterAddress::from_hex(&addr) + .map_err(|e| PyValueError::new_err(format!("Failed to parse address: {e}")))?; + + future_into_py(py, async move { + let data = client + .register_get(&addr) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get register: {e}")))?; + + Ok(data) + }) + } + + /// Get the current value of the register + fn register_cost<'a>(&self, py: Python<'a>, owner: PyPublicKey) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let cost = client.register_cost(&owner.inner).await.map_err(|e| { + PyRuntimeError::new_err(format!("Failed to get register cost: {e}")) + })?; + + Ok(cost.to_string()) + }) + } + + /// Retrieves and returns a decrypted vault if one exists. + /// + /// Returns the content type of the bytes in the vault. + fn fetch_and_decrypt_vault<'a>( + &self, + py: Python<'a>, + key: &PyVaultSecretKey, + ) -> PyResult> { + let client = self.inner.clone(); + let key = key.inner.clone(); + + future_into_py(py, async move { + match client.fetch_and_decrypt_vault(&key).await { + Ok((data, content_type)) => Ok((data.to_vec(), content_type)), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to fetch vault: {e}" + ))), + } + }) + } + + /// Get the user data from the vault + fn get_user_data_from_vault<'a>( + &self, + py: Python<'a>, + key: &PyVaultSecretKey, + ) -> PyResult> { + let client = self.inner.clone(); + let key = key.inner.clone(); + + future_into_py(py, async move { + match client.get_user_data_from_vault(&key).await { + Ok(user_data) => Ok(PyUserData { inner: user_data }), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to get user data from vault: {e}" + ))), + } + }) + } + + /// Put the user data to the vault. + /// + /// Returns the total cost of the put operation. + fn put_user_data_to_vault<'a>( + &self, + py: Python<'a>, + key: &PyVaultSecretKey, + payment: &PyPaymentOption, + user_data: &PyUserData, + ) -> PyResult> { + let client = self.inner.clone(); + let key = key.inner.clone(); + let payment = payment.inner.clone(); + let user_data = user_data.inner.clone(); + + future_into_py(py, async move { + match client + .put_user_data_to_vault(&key, payment, user_data) + .await + { + Ok(cost) => Ok(cost.to_string()), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to put user data: {e}" + ))), + } + }) + } + + /// Get a pointer from the network + fn pointer_get<'a>( + &self, + py: Python<'a>, + addr: PyPointerAddress, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + match client.pointer_get(&addr.inner).await { + Ok(pointer) => Ok(PyPointer { inner: pointer }), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to get pointer: {e}" + ))), + } + }) + } + + /// Check if a pointer exists on the network + fn pointer_check_existance<'a>( + &self, + py: Python<'a>, + addr: PyPointerAddress, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + let exists = client + .pointer_check_existance(&addr.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get pointer: {e}")))?; + + Ok(exists) + }) + } + + /// Manually store a pointer on the network + fn pointer_put<'a>( + &self, + py: Python<'a>, + pointer: &PyPointer, + payment_option: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let pointer = pointer.inner.clone(); + let payment = payment_option.inner.clone(); + + future_into_py(py, async move { + let (_cost, addr) = client + .pointer_put(pointer, payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to put pointer: {e}")))?; + Ok(PyPointerAddress { inner: addr }) + }) + } + + /// Create a new pointer on the network. + /// + /// Make sure that the owner key is not already used for another pointer as each key is associated with one pointer + fn pointer_create<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + target: PyPointerTarget, + payment_option: &PyPaymentOption, + ) -> PyResult> { + let client = self.inner.clone(); + let payment = payment_option.inner.clone(); + + future_into_py(py, async move { + let (cost, addr) = client + .pointer_create(&owner.inner, target.inner, payment) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to create pointer: {e}")))?; + + Ok((cost.to_string(), PyPointerAddress { inner: addr })) + }) + } + + /// Update an existing pointer to point to a new target on the network. + /// + /// The pointer needs to be created first with `pointer_put`. + /// This operation is free as the pointer was already paid for at creation. + /// Only the latest version of the pointer is kept on the Network, previous versions will be overwritten and unrecoverable. + fn pointer_update<'a>( + &self, + py: Python<'a>, + owner: PySecretKey, + target: PyPointerTarget, + ) -> PyResult> { + let client = self.inner.clone(); + + future_into_py(py, async move { + client + .pointer_update(&owner.inner, target.inner) + .await + .map_err(|e| PyRuntimeError::new_err(format!("Failed to update pointer: {e}")))?; + + Ok(()) + }) + } + + /// Calculate the cost of storing a pointer + fn pointer_cost<'a>(&self, py: Python<'a>, key: &PyPublicKey) -> PyResult> { + let client = self.inner.clone(); + let key = key.inner; + + future_into_py(py, async move { + match client.pointer_cost(&key).await { + Ok(cost) => Ok(cost.to_string()), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to get pointer cost: {e}" + ))), + } + }) + } +} + +/// A network address where a pointer is stored. +/// The address is derived from the owner's public key. +#[pyclass(name = "PointerAddress")] +#[derive(Debug, Clone)] +pub struct PyPointerAddress { + inner: PointerAddress, +} + +#[pymethods] +impl PyPointerAddress { + /// Initialise pointer address from hex string. + #[staticmethod] + pub fn from_hex(hex: String) -> PyResult { + let bytes = hex::decode(hex) + .map_err(|e| PyValueError::new_err(format!("`hex` not a valid hex string: {e}")))?; + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| PyValueError::new_err("`hex` invalid: must be 32 bytes"))?; + + Ok(Self { + inner: PointerAddress::new(XorName(bytes)), + }) + } + + /// Returns the hex string representation of the pointer address. + #[getter] + pub fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } +} + +/// Pointer, a mutable address pointing to other data on the Network. +/// It is stored at the owner's public key and can only be updated by the owner. +#[pyclass(name = "Pointer")] +#[derive(Debug, Clone)] +pub struct PyPointer { + inner: Pointer, +} + +#[pymethods] +impl PyPointer { + /// Create a new pointer, signing it with the provided secret key. + /// This pointer would be stored on the network at the provided key's public key. + /// There can only be one pointer at a time at the same address (one per key). + #[new] + pub fn new(key: &PySecretKey, counter: u32, target: &PyPointerTarget) -> PyResult { + Ok(Self { + inner: Pointer::new(&key.inner, counter, target.inner.clone()), + }) + } + + /// Returns the network address where this pointer is stored. + pub fn address(&self) -> PyPointerAddress { + PyPointerAddress { + inner: self.inner.address(), + } + } + + /// Returns the hex string representation of the pointer's target. + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } + + /// Returns the target that this pointer points to. + #[getter] + fn target(&self) -> PyPointerTarget { + PyPointerTarget { + inner: PointerTarget::ChunkAddress(ChunkAddress::new(self.inner.xorname())), + } + } + + fn __str__(&self) -> PyResult { + Ok(self.hex()) + } +} + +/// The target that a pointer points to on the network. +#[pyclass(name = "PointerTarget")] +#[derive(Debug, Clone)] +pub struct PyPointerTarget { + inner: PointerTarget, } #[pymethods] -impl PyClient { +impl PyPointerTarget { + /// Initialize a pointer target from a chunk address hex string. #[staticmethod] - fn connect(peers: Vec) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let peers = peers - .into_iter() - .map(|addr| addr.parse()) - .collect::, _>>() - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Invalid multiaddr: {e}")) - })?; - - let client = rt - .block_on(RustClient::init_with_peers(peers)) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to connect: {e}")) - })?; + fn from_hex(hex: &str) -> PyResult { + let bytes = hex::decode(hex) + .map_err(|e| PyValueError::new_err(format!("`hex` not a valid hex string: {e}")))?; + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| PyValueError::new_err("`hex` invalid: must be 32 bytes"))?; - Ok(Self { inner: client }) + Ok(Self { + inner: PointerTarget::ChunkAddress(ChunkAddress::new(XorName(bytes))), + }) } - fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let access = rt - .block_on( - self.inner - .data_put(Bytes::from(data), payment.inner.clone()), - ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}")) - })?; + /// Returns the hex string representation of this pointer address. + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } - Ok(PyDataMapChunk { inner: access }) + #[getter] + fn target(&self) -> PyPointerTarget { + PyPointerTarget { + inner: PointerTarget::ChunkAddress(ChunkAddress::new(self.inner.xorname())), + } } - fn data_get(&self, access: &PyDataMapChunk) -> PyResult> { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let data = rt - .block_on(self.inner.data_get(access.inner.clone())) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get private data: {e}")) - })?; - Ok(data.to_vec()) - } - - fn data_put_public(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let addr = rt - .block_on( - self.inner - .data_put_public(bytes::Bytes::from(data), payment.inner.clone()), - ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to put data: {e}")) - })?; + /// Creates a pointer target from a chunk address. + #[staticmethod] + fn from_chunk_address(addr: &PyChunkAddress) -> Self { + Self { + inner: PointerTarget::ChunkAddress(addr.inner), + } + } - Ok(crate::client::address::addr_to_str(addr)) + fn __str__(&self) -> PyResult { + Ok(self.hex()) } +} - fn data_get_public(&self, addr: &str) -> PyResult> { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let addr = crate::client::address::str_to_addr(addr).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {e}")) - })?; +/// An address of a chunk of data on the network. Used to locate and retrieve data chunks. +#[pyclass(name = "ChunkAddress")] +#[derive(Debug, Clone)] +pub struct PyChunkAddress { + inner: ChunkAddress, +} - let data = rt.block_on(self.inner.data_get_public(addr)).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get data: {e}")) - })?; +impl From for PyChunkAddress { + fn from(addr: ChunkAddress) -> Self { + Self { inner: addr } + } +} - Ok(data.to_vec()) +impl From for ChunkAddress { + fn from(addr: PyChunkAddress) -> Self { + addr.inner } +} - fn vault_cost(&self, key: &PyVaultSecretKey) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let cost = rt - .block_on(self.inner.vault_cost(&key.inner)) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get vault cost: {e}")) - })?; - Ok(cost.to_string()) +#[pymethods] +impl PyChunkAddress { + /// Creates a new chunk address from a string representation. + #[new] + fn new(#[pyo3(from_py_with = "str_to_addr")] addr: XorName) -> PyResult { + Ok(Self { + inner: ChunkAddress::new(addr), + }) } - fn write_bytes_to_vault( - &self, - data: Vec, - payment: &PyPaymentOption, - key: &PyVaultSecretKey, - content_type: u64, - ) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let cost = rt - .block_on(self.inner.write_bytes_to_vault( - bytes::Bytes::from(data), - payment.inner.clone(), - &key.inner, - content_type, - )) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to write to vault: {e}")) - })?; - Ok(cost.to_string()) + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) } - fn fetch_and_decrypt_vault(&self, key: &PyVaultSecretKey) -> PyResult<(Vec, u64)> { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let (data, content_type) = rt - .block_on(self.inner.fetch_and_decrypt_vault(&key.inner)) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to fetch vault: {e}")) - })?; - Ok((data.to_vec(), content_type)) + /// Creates a chunk address from a hex string representation. + #[staticmethod] + fn from_chunk_address(addr: &str) -> PyResult { + let bytes = + hex::decode(addr).map_err(|e| PyValueError::new_err(format!("`addr` invalid: {e}")))?; + + if bytes.len() != 32 { + return Err(PyValueError::new_err("`addr` invalid: must be 32 bytes")); + } + + let mut xorname = [0u8; 32]; + xorname.copy_from_slice(&bytes); + + Ok(Self { + inner: ChunkAddress::new(XorName(xorname)), + }) } - fn get_user_data_from_vault(&self, key: &PyVaultSecretKey) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let user_data = rt - .block_on(self.inner.get_user_data_from_vault(&key.inner)) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get user data: {e}")) - })?; - Ok(PyUserData { inner: user_data }) + fn __str__(&self) -> PyResult { + Ok(self.hex()) } - fn put_user_data_to_vault( - &self, - key: &PyVaultSecretKey, - payment: &PyPaymentOption, - user_data: &PyUserData, - ) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let cost = rt - .block_on(self.inner.put_user_data_to_vault( - &key.inner, - payment.inner.clone(), - user_data.inner.clone(), - )) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to put user data: {e}")) - })?; - Ok(cost.to_string()) + fn __repr__(&self) -> PyResult { + Ok(format!("ChunkAddress({})", self.hex())) } } +/// A wallet for interacting with the network's payment system. +/// Handles token transfers, balance checks, and payments for network operations. #[pyclass(name = "Wallet")] -pub(crate) struct PyWallet { - inner: RustWallet, +#[derive(Clone)] +pub struct PyWallet { + pub(crate) inner: Wallet, } #[pymethods] impl PyWallet { + /// Creates a new wallet from a private key string. + /// The wallet will be configured to use the ArbitrumOne network. #[new] fn new(private_key: String) -> PyResult { - let wallet = RustWallet::new_from_private_key( + let wallet = Wallet::new_from_private_key( Network::ArbitrumOne, // TODO: Make this configurable &private_key, ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Invalid private key: {e}")) - })?; + .map_err(|e| PyValueError::new_err(format!("`private_key` invalid: {e}")))?; Ok(Self { inner: wallet }) } + /// Creates a new wallet from a private key string with a specified network. + #[staticmethod] + fn new_from_private_key(network: PyNetwork, private_key: &str) -> PyResult { + let inner = Wallet::new_from_private_key(network.inner, private_key) + .map_err(|e| PyValueError::new_err(format!("`private_key` invalid: {e}")))?; + + Ok(Self { inner }) + } + + /// Returns a string representation of the wallet's address. fn address(&self) -> String { format!("{:?}", self.inner.address()) } - fn balance(&self) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let balance = rt - .block_on(async { self.inner.balance_of_tokens().await }) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get balance: {e}")) - })?; - - Ok(balance.to_string()) + /// Returns the raw balance of payment tokens in the wallet. + fn balance<'a>(&self, py: Python<'a>) -> PyResult> { + let client = self.inner.clone(); + future_into_py(py, async move { + match client.balance_of_tokens().await { + Ok(balance) => Ok(balance.to_string()), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to get balance: {e}" + ))), + } + }) } - fn balance_of_gas(&self) -> PyResult { - let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); - let balance = rt - .block_on(async { self.inner.balance_of_gas_tokens().await }) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get balance: {e}")) - })?; - - Ok(balance.to_string()) + /// Returns the current balance of gas tokens in the wallet. + fn balance_of_gas<'a>(&self, py: Python<'a>) -> PyResult> { + let client = self.inner.clone(); + future_into_py(py, async move { + match client.balance_of_gas_tokens().await { + Ok(balance) => Ok(balance.to_string()), + Err(e) => Err(PyRuntimeError::new_err(format!( + "Failed to get balance: {e}" + ))), + } + }) } } +/// Options for making payments on the network. #[pyclass(name = "PaymentOption")] -pub(crate) struct PyPaymentOption { - inner: RustPaymentOption, +#[derive(Clone)] +pub struct PyPaymentOption { + pub(crate) inner: PaymentOption, } #[pymethods] impl PyPaymentOption { + /// Creates a payment option using the provided wallet. #[staticmethod] fn wallet(wallet: &PyWallet) -> Self { Self { - inner: RustPaymentOption::Wallet(wallet.inner.clone()), + inner: PaymentOption::Wallet(wallet.inner.clone()), + } + } +} + +/// A cryptographic secret key used for signing operations. +/// Can be used to derive a public key and perform cryptographic operations. +#[pyclass(name = "SecretKey")] +#[derive(Debug, Clone)] +pub struct PySecretKey { + inner: SecretKey, +} + +#[pymethods] +impl PySecretKey { + /// Creates a new random secret key. + #[new] + fn new() -> PyResult { + Ok(Self { + inner: SecretKey::random(), + }) + } + + /// Creates a secret key from a hex string representation. + #[staticmethod] + fn from_hex(hex_str: &str) -> PyResult { + SecretKey::from_hex(hex_str) + .map(|key| Self { inner: key }) + .map_err(|e| PyValueError::new_err(format!("Invalid hex key: {e}"))) + } + + /// Derives and returns the corresponding public key. + fn public_key(&self) -> PyPublicKey { + PyPublicKey { + inner: self.inner.public_key(), } } + + /// Returns the hex string representation of the key. + fn to_hex(&self) -> String { + self.inner.to_hex() + } +} + +/// A cryptographic public key derived from a secret key. +#[pyclass(name = "PublicKey")] +#[derive(Debug, Clone)] +pub struct PyPublicKey { + inner: PublicKey, +} + +#[pymethods] +impl PyPublicKey { + /// Creates a new random public key by generating a random secret key. + #[new] + fn new() -> PyResult { + let secret = SecretKey::random(); + Ok(Self { + inner: secret.public_key(), + }) + } + + /// Creates a public key from a hex string representation. + #[staticmethod] + fn from_hex(hex_str: &str) -> PyResult { + PublicKey::from_hex(hex_str) + .map(|key| Self { inner: key }) + .map_err(|e| PyValueError::new_err(format!("Invalid hex key: {e}"))) + } + + fn to_hex(&self) -> String { + self.inner.to_hex() + } } +/// A secret key used to encrypt and decrypt vault data. #[pyclass(name = "VaultSecretKey")] -pub(crate) struct PyVaultSecretKey { +#[derive(Debug, Clone)] +pub struct PyVaultSecretKey { inner: VaultSecretKey, } #[pymethods] impl PyVaultSecretKey { + /// Creates a new random vault secret key. #[new] fn new() -> PyResult { Ok(Self { @@ -242,7 +1425,7 @@ impl PyVaultSecretKey { fn from_hex(hex_str: &str) -> PyResult { VaultSecretKey::from_hex(hex_str) .map(|key| Self { inner: key }) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex key: {e}"))) + .map_err(|e| PyValueError::new_err(format!("Invalid hex key: {e}"))) } fn to_hex(&self) -> String { @@ -250,13 +1433,19 @@ impl PyVaultSecretKey { } } +/// UserData is stored in Vaults and contains most of a user's private data: +/// It allows users to keep track of only the key to their User Data Vault +/// while having the rest kept on the Network encrypted in a Vault for them +/// Using User Data Vault is optional, one can decide to keep all their data locally instead. #[pyclass(name = "UserData")] -pub(crate) struct PyUserData { +#[derive(Debug, Clone)] +pub struct PyUserData { inner: UserData, } #[pymethods] impl PyUserData { + /// Creates a new empty UserData instance. #[new] fn new() -> Self { Self { @@ -264,29 +1453,16 @@ impl PyUserData { } } - fn add_file_archive(&mut self, archive: &str) -> Option { - let name = XorName::from_content(archive.as_bytes()); - let archive_addr = ArchiveAddr::from_content(&name); - self.inner.add_file_archive(archive_addr) - } - - fn add_private_file_archive(&mut self, archive: &str) -> Option { - let name = XorName::from_content(archive.as_bytes()); - let private_access = match PrivateArchiveAccess::from_hex(&name.to_string()) { - Ok(access) => access, - Err(_e) => return None, - }; - self.inner.add_private_file_archive(private_access) - } - + /// Returns a list of public file archives as (address, name) pairs. fn file_archives(&self) -> Vec<(String, String)> { self.inner .file_archives .iter() - .map(|(addr, name)| (format!("{addr:x}"), name.clone())) + .map(|(addr, name)| (hex::encode(addr), name.clone())) .collect() } + /// Returns a list of private file archives as (data_map, name) pairs. fn private_file_archives(&self) -> Vec<(String, String)> { self.inner .private_file_archives @@ -296,25 +1472,31 @@ impl PyUserData { } } +/// A map with encrypted data pieces on the network. Used to locate and reconstruct private data. #[pyclass(name = "DataMapChunk")] -#[derive(Clone)] -pub(crate) struct PyDataMapChunk { +#[derive(Debug, Clone)] +pub struct PyDataMapChunk { inner: DataMapChunk, } #[pymethods] impl PyDataMapChunk { + /// Creates a DataMapChunk from a hex string representation. #[staticmethod] fn from_hex(hex: &str) -> PyResult { DataMapChunk::from_hex(hex) .map(|access| Self { inner: access }) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex: {e}"))) + .map_err(|e| PyValueError::new_err(format!("Invalid hex: {e}"))) } + /// Returns the hex string representation of this DataMapChunk. fn to_hex(&self) -> String { self.inner.to_hex() } + /// Returns the private address of this DataMapChunk. + /// + /// Note that this is not a network address, it is only used for refering to private data client side. fn address(&self) -> String { self.inner.address().to_string() } @@ -323,10 +1505,10 @@ impl PyDataMapChunk { #[pyfunction] fn encrypt(data: Vec) -> PyResult<(Vec, Vec>)> { let (data_map, chunks) = self_encryption::encrypt(Bytes::from(data)) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Encryption failed: {e}")))?; + .map_err(|e| PyRuntimeError::new_err(format!("Encryption failed: {e}")))?; let data_map_bytes = rmp_serde::to_vec(&data_map) - .map_err(|e| PyValueError::new_err(format!("Failed to serialize data map: {e}")))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to serialize data map: {e}")))?; let chunks_bytes: Vec> = chunks .into_iter() @@ -336,15 +1518,358 @@ fn encrypt(data: Vec) -> PyResult<(Vec, Vec>)> { Ok((data_map_bytes, chunks_bytes)) } +#[pyclass(name = "Network")] +#[derive(Debug, Clone)] +pub struct PyNetwork { + inner: Network, +} + +#[pymethods] +impl PyNetwork { + /// Creates a new network configuration. + /// + /// If `local` is true, configures for local network connections. + #[new] + fn new(local: bool) -> PyResult { + let inner = Network::new(local).map_err(|e| PyRuntimeError::new_err(format!("{e:?}")))?; + Ok(Self { inner }) + } +} + +/// Metadata for files in an archive, containing creation time, modification time, and size. +#[pyclass(name = "Metadata")] +#[derive(Debug, Clone)] +pub struct PyMetadata { + inner: Metadata, +} + +#[pymethods] +impl PyMetadata { + /// Create new metadata with the given file size + #[new] + fn new(size: u64) -> Self { + Self { + inner: Metadata::new_with_size(size), + } + } + + /// Get the creation time as Unix timestamp in seconds + #[getter] + fn get_created(&self) -> u64 { + self.inner.created + } + + /// Set the creation time as Unix timestamp in seconds + #[setter] + fn set_created(&mut self, value: u64) { + self.inner.created = value; + } + + /// Get the modification time as Unix timestamp in seconds + #[getter] + fn get_modified(&self) -> u64 { + self.inner.modified + } + + /// Set the modification time as Unix timestamp in seconds + #[setter] + fn set_modified(&mut self, value: u64) { + self.inner.modified = value; + } + + /// Get the file size in bytes + #[getter] + fn get_size(&self) -> u64 { + self.inner.size + } + + /// Set the file size in bytes + #[setter] + fn set_size(&mut self, value: u64) { + self.inner.size = value; + } +} + +/// A public archive containing files that can be accessed by anyone on the network. +#[pyclass(name = "PublicArchive")] +#[derive(Debug, Clone)] +pub struct PyPublicArchive { + inner: PublicArchive, +} + +#[pymethods] +impl PyPublicArchive { + /// Create a new empty archive + #[new] + fn new() -> Self { + Self { + inner: PublicArchive::new(), + } + } + + /// Rename a file in the archive. + /// + /// Returns None on success, or error message on failure + fn rename_file(&mut self, old_path: PathBuf, new_path: PathBuf) -> PyResult<()> { + self.inner + .rename_file(&old_path, &new_path) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to rename file: {e}"))) + } + + /// Add a file to the archive + fn add_file( + &mut self, + path: PathBuf, + #[pyo3(from_py_with = "str_to_addr")] addr: XorName, + metadata: &PyMetadata, + ) { + self.inner.add_file(path, addr, metadata.inner.clone()); + } + + /// List all files in the archive. + /// + /// Returns a list of (path, metadata) tuples + fn files(&self) -> Vec<(PathBuf, PyMetadata)> { + self.inner + .files() + .into_iter() + .map(|(path, meta)| (path, PyMetadata { inner: meta })) + .collect() + } + + /// List all data addresses of files in the archive + fn addresses(&self) -> Vec { + self.inner + .addresses() + .into_iter() + .map(crate::client::address::addr_to_str) + .collect() + } +} + +/// A public archive containing files that can be accessed by anyone on the network. +#[pyclass(name = "PrivateArchive")] +#[derive(Debug, Clone)] +pub struct PyPrivateArchive { + inner: PrivateArchive, +} + +#[pymethods] +impl PyPrivateArchive { + /// Create a new empty archive + #[new] + fn new() -> Self { + Self { + inner: PrivateArchive::new(), + } + } + + /// Rename a file in the archive. + /// + /// Returns None on success, or error message on failure + fn rename_file(&mut self, old_path: PathBuf, new_path: PathBuf) -> PyResult<()> { + self.inner + .rename_file(&old_path, &new_path) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to rename file: {e}"))) + } + + /// Add a file to a local archive. Note that this does not upload the archive to the network. + fn add_file(&mut self, path: PathBuf, data_map: &PyDataMapChunk, metadata: &PyMetadata) { + self.inner + .add_file(path, data_map.inner.clone(), metadata.inner.clone()); + } + + /// List all files in the archive. + fn files(&self) -> Vec<(PathBuf, PyMetadata)> { + self.inner + .files() + .into_iter() + .map(|(path, meta)| (path, PyMetadata { inner: meta })) + .collect() + } + + /// List all data maps of files in the archive + fn data_maps(&self) -> Vec { + self.inner + .data_maps() + .into_iter() + .map(|data_map| PyDataMapChunk { inner: data_map }) + .collect() + } +} + +/// A generic GraphEntry on the Network. +/// +/// Graph entries are stored at the owner's public key. Note that there can only be one graph entry per owner. +/// Graph entries can be linked to other graph entries as parents or descendants. +/// Applications are free to define the meaning of these links, those are not enforced by the protocol. +/// The protocol only ensures that the graph entry is immutable once uploaded and that the signature is valid and matches the owner. +/// +/// For convenience it is advised to make use of BLS key derivation to create multiple graph entries from a single key. +#[pyclass(name = "GraphEntry")] +#[derive(Debug, Clone)] +pub struct PyGraphEntry { + inner: GraphEntry, +} + +/// Scratchpad, a mutable space for encrypted data on the Network +#[pyclass(name = "Scratchpad")] +#[derive(Debug, Clone)] +pub struct PyScratchpad { + inner: Scratchpad, +} + +/// A handle to the register history +#[pyclass(name = "RegisterHistory")] +#[derive(Clone)] +pub struct PyRegisterHistory { + inner: Arc>, +} + +impl PyRegisterHistory { + fn new(history: RegisterHistory) -> Self { + Self { + inner: Arc::new(futures::lock::Mutex::new(history)), + } + } +} + +#[pymethods] +impl PyRegisterHistory { + fn next<'a>(&'a mut self, py: Python<'a>) -> PyResult> { + let arc = Arc::clone(&self.inner); + + future_into_py(py, async move { + let mut register_history = arc.lock().await; + let value = register_history + .next() + .await + .map_err(|e| PyRuntimeError::new_err(format!("history `next` failed: {e}")))?; + + Ok(value) + }) + } + + fn collect<'a>(&'a mut self, py: Python<'a>) -> PyResult> { + let arc = Arc::clone(&self.inner); + + future_into_py(py, async move { + let mut register_history = arc.lock().await; + let values = register_history + .collect() + .await + .map_err(|e| PyRuntimeError::new_err(format!("history `collect` failed: {e}")))?; + + Ok(values) + }) + } +} + +/// Configuration for the `Client` which can be provided through: `init_with_config`. +#[pyclass(name = "ClientConfig")] +#[derive(Debug, Clone)] +pub struct PyClientConfig { + inner: ClientConfig, +} + +#[pymethods] +impl PyClientConfig { + #[staticmethod] + fn new() -> Self { + Self { + inner: ClientConfig::default(), + } + } + + /// Whether we're expected to connect to a local network. + #[getter] + fn get_local(&self) -> bool { + self.inner.local + } + + /// Whether we're expected to connect to a local network. + #[setter] + fn set_local(&mut self, value: bool) { + self.inner.local = value; + } + + /// List of peers to connect to. + /// + /// If not provided, the client will use the default bootstrap peers. + #[getter] + fn get_peers(&self) -> Option> { + self.inner + .peers + .as_ref() + .map(|peers| peers.iter().map(|p| p.to_string()).collect()) + } + + /// List of peers to connect to. If given empty list, the client will use the default bootstrap peers. + #[setter] + fn set_peers(&mut self, peers: Vec) -> PyResult<()> { + if peers.is_empty() { + self.inner.peers = None; + return Ok(()); + } + + let peers: Vec = peers + .iter() + .map(|p| Multiaddr::from_str(p)) + .collect::>() + .map_err(|e| PyValueError::new_err(format!("Failed to parse peers: {e}")))?; + + self.inner.peers = Some(peers); + Ok(()) + } + + /// EVM network to use for quotations and payments. + #[getter] + fn get_network(&self) -> PyNetwork { + PyNetwork { + inner: self.inner.evm_network.clone(), + } + } + + /// EVM network to use for quotations and payments. + #[setter] + fn set_network(&mut self, network: PyNetwork) { + self.inner.evm_network = network.inner; + } + + // TODO + // fn strategy() { } +} + #[pymodule] #[pyo3(name = "autonomi_client")] -fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn autonomi_client_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(encrypt, m)?)?; Ok(()) } + +// Helper function to convert argument hex string to XorName. +fn str_to_addr(addr: &Bound<'_, PyAny>) -> PyResult { + let addr: String = addr.extract()?; + crate::client::address::str_to_addr(&addr) + .map_err(|e| PyValueError::new_err(format!("`addr` has invalid format: {e:?}"))) +} diff --git a/autonomi/src/self_encryption.rs b/autonomi/src/self_encryption.rs index 30f7454457..e15213ab11 100644 --- a/autonomi/src/self_encryption.rs +++ b/autonomi/src/self_encryption.rs @@ -8,6 +8,7 @@ use ant_protocol::storage::Chunk; use bytes::{BufMut, Bytes, BytesMut}; +use rayon::prelude::*; use self_encryption::{DataMap, MAX_CHUNK_SIZE}; use serde::{Deserialize, Serialize}; use tracing::debug; @@ -30,14 +31,14 @@ pub(crate) enum DataMapLevel { Additional(DataMap), } -pub(crate) fn encrypt(data: Bytes) -> Result<(Chunk, Vec), Error> { +pub fn encrypt(data: Bytes) -> Result<(Chunk, Vec), Error> { let (data_map, chunks) = self_encryption::encrypt(data)?; let (data_map_chunk, additional_chunks) = pack_data_map(data_map)?; // Transform `EncryptedChunk` into `Chunk` let chunks: Vec = chunks - .into_iter() - .map(|c| Chunk::new(c.content)) + .into_par_iter() + .map(|c| Chunk::new(c.content.clone())) .chain(additional_chunks) .collect(); @@ -58,7 +59,7 @@ fn pack_data_map(data_map: DataMap) -> Result<(Chunk, Vec), Error> { debug!("Max chunk size: {}", *MAX_CHUNK_SIZE); let chunk = Chunk::new(chunk_content); // If datamap chunk is less than `MAX_CHUNK_SIZE` return it so it can be directly sent to the network. - if *MAX_CHUNK_SIZE >= chunk.serialised_size() { + if *MAX_CHUNK_SIZE >= chunk.size() { chunks.reverse(); // Returns the last datamap, and all the chunks produced. break (chunk, chunks); diff --git a/autonomi/tests/chunk.rs b/autonomi/tests/chunk.rs new file mode 100644 index 0000000000..45881cbc2f --- /dev/null +++ b/autonomi/tests/chunk.rs @@ -0,0 +1,45 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_logging::LogBuilder; +use autonomi::client::payment::PaymentOption; +use autonomi::{client::chunk::Chunk, Bytes, Client}; +use eyre::Result; +use serial_test::serial; +use test_utils::evm::get_funded_wallet; + +#[tokio::test] +#[serial] +async fn chunk_put_manual() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("chunk", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let chunk = Chunk::new(Bytes::from("Hello, world!")); + + // estimate the cost of the chunk + let cost = client.chunk_cost(chunk.address()).await?; + println!("chunk cost: {cost}"); + + // put the chunk + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client.chunk_put(&chunk, payment_option).await?; + assert_eq!(addr, *chunk.address()); + println!("chunk put 1 cost: {cost}"); + + // wait for the chunk to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the chunk is stored + let got = client.chunk_get(&addr).await?; + assert_eq!(got, chunk.clone()); + println!("chunk got 1"); + + Ok(()) +} diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index 755a1cac8f..8359c6ab90 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -4,13 +4,15 @@ use alloy::network::TransactionBuilder; use alloy::providers::Provider; use ant_evm::{QuoteHash, TxHash}; use ant_logging::LogBuilder; +use ant_protocol::storage::DataTypes; use autonomi::client::external_signer::encrypt_data; -use autonomi::client::files::archive::{Metadata, PrivateArchive}; +use autonomi::client::files::{archive_private::PrivateArchive, Metadata}; use autonomi::client::payment::{receipt_from_store_quotes, Receipt}; use autonomi::client::quote::StoreQuote; use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; use autonomi::client::vault::VaultSecretKey; -use autonomi::{Client, Wallet}; +use autonomi::vault::UserData; +use autonomi::{Client, Scratchpad, Wallet}; use bytes::Bytes; use std::collections::BTreeMap; use std::time::Duration; @@ -23,22 +25,23 @@ async fn pay_for_data(client: &Client, wallet: &Wallet, data: Bytes) -> eyre::Re let (data_map_chunk, chunks) = encrypt_data(data)?; let map_xor_name = *data_map_chunk.address().xorname(); - let mut xor_names = vec![map_xor_name]; + let mut xor_names = vec![(map_xor_name, data_map_chunk.size())]; for chunk in chunks { - xor_names.push(*chunk.name()); + xor_names.push((*chunk.name(), chunk.size())); } - pay_for_content_addresses(client, wallet, xor_names.into_iter()).await + pay_for_content_addresses(client, wallet, DataTypes::Chunk, xor_names.into_iter()).await } async fn pay_for_content_addresses( client: &Client, wallet: &Wallet, - content_addrs: impl Iterator + Clone, + data_types: DataTypes, + content_addrs: impl Iterator + Clone, ) -> eyre::Result { let (quotes, quote_payments, _free_chunks) = client - .get_quotes_for_content_addresses(content_addrs) + .get_quotes_for_content_addresses(data_types, content_addrs) .await?; // Form quotes payment transaction data @@ -111,7 +114,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_data_access = client.data_put(data.clone(), receipt.into()).await?; + let (_cost, private_data_access) = client.data_put(data.clone(), receipt.into()).await?; let mut private_archive = PrivateArchive::new(); private_archive.add_file( @@ -126,34 +129,32 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_archive_access = client.archive_put(&private_archive, receipt.into()).await?; + let (_cost, private_archive_access) = + client.archive_put(&private_archive, receipt.into()).await?; let vault_key = VaultSecretKey::random(); - let mut user_data = client - .get_user_data_from_vault(&vault_key) - .await - .unwrap_or_default(); + let mut user_data = UserData::default(); - user_data.add_private_file_archive_with_name( - private_archive_access.clone(), - "test-archive".to_string(), - ); - - let (scratch, is_new) = client - .get_or_create_scratchpad(&vault_key, *USER_DATA_VAULT_CONTENT_IDENTIFIER) - .await?; + user_data + .add_private_file_archive_with_name(private_archive_access, "test-archive".to_string()); - assert!(is_new, "Scratchpad is not new"); + let scratchpad = Scratchpad::new( + &vault_key, + *USER_DATA_VAULT_CONTENT_IDENTIFIER, + &user_data.to_bytes()?, + 0, + ); - let scratch_addresses = if is_new { - scratch.to_xor_name_vec() - } else { - vec![] - }; + let scratch_addresses = vec![(scratchpad.xorname(), scratchpad.payload_size())]; - let receipt = - pay_for_content_addresses(&client, &wallet, scratch_addresses.into_iter()).await?; + let receipt = pay_for_content_addresses( + &client, + &wallet, + DataTypes::Scratchpad, + scratch_addresses.into_iter(), + ) + .await?; sleep(Duration::from_secs(5)).await; @@ -170,7 +171,7 @@ async fn external_signer_put() -> eyre::Result<()> { .expect("No private archive present in the UserData") .clone(); - let fetched_private_archive = client.archive_get(fetched_private_archive_access).await?; + let fetched_private_archive = client.archive_get(&fetched_private_archive_access).await?; let (_, (fetched_private_file_access, _)) = fetched_private_archive .map() @@ -178,7 +179,7 @@ async fn external_signer_put() -> eyre::Result<()> { .next() .expect("No file present in private archive"); - let fetched_private_file = client.data_get(fetched_private_file_access.clone()).await?; + let fetched_private_file = client.data_get(fetched_private_file_access).await?; assert_eq!( fetched_private_file, data, diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 926baeb4fd..ebeba12f7a 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -6,11 +6,10 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -#![cfg(feature = "fs")] - use ant_logging::LogBuilder; use autonomi::Client; use eyre::Result; +use serial_test::serial; use sha2::{Digest, Sha256}; use std::fs::File; use std::io::{BufReader, Read}; @@ -20,8 +19,9 @@ use tokio::time::sleep; use walkdir::WalkDir; // With a local evm network, and local network, run: -// EVM_NETWORK=local cargo test --features="fs,local" --package autonomi --test file +// EVM_NETWORK=local cargo test --package autonomi --test fs #[tokio::test] +#[serial] async fn dir_upload_download() -> Result<()> { let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("dir_upload_download", false); @@ -29,14 +29,14 @@ async fn dir_upload_download() -> Result<()> { let client = Client::init_local().await?; let wallet = get_funded_wallet(); - let addr = client + let (_cost, addr) = client .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(10)).await; client - .dir_download_public(addr, "tests/file/test_dir_fetched".into()) + .dir_download_public(&addr, "tests/file/test_dir_fetched".into()) .await?; // compare the two directories @@ -76,8 +76,8 @@ fn compute_dir_sha256(dir: &str) -> Result { Ok(format!("{:x}", hasher.finalize())) } -#[cfg(feature = "vault")] #[tokio::test] +#[serial] async fn file_into_vault() -> Result<()> { let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("file", false); @@ -85,12 +85,12 @@ async fn file_into_vault() -> Result<()> { let wallet = get_funded_wallet(); let client_sk = bls::SecretKey::random(); - let addr = client + let (_cost, addr) = client .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(2)).await; - let archive = client.archive_get_public(addr).await?; + let archive = client.archive_get_public(&addr).await?; let set_version = 0; client .write_bytes_to_vault(archive.to_bytes()?, wallet.into(), &client_sk, set_version) diff --git a/autonomi/tests/graph.rs b/autonomi/tests/graph.rs new file mode 100644 index 0000000000..c2f3a87465 --- /dev/null +++ b/autonomi/tests/graph.rs @@ -0,0 +1,83 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_logging::LogBuilder; +use autonomi::{ + client::{ + graph::{GraphEntry, GraphError}, + payment::PaymentOption, + }, + Client, +}; +use eyre::Result; +use serial_test::serial; +use test_utils::evm::get_funded_wallet; + +#[tokio::test] +#[serial] +async fn graph_entry_put() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("graph_entry", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let content = [0u8; 32]; + let graph_entry = GraphEntry::new(&key, vec![], content, vec![]); + + // estimate the cost of the graph_entry + let cost = client.graph_entry_cost(&key.public_key()).await?; + println!("graph_entry cost: {cost}"); + + // put the graph_entry + let payment_option = PaymentOption::from(&wallet); + client + .graph_entry_put(graph_entry.clone(), payment_option) + .await?; + println!("graph_entry put 1"); + + // wait for the graph_entry to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the graph_entry is stored + let txs = client.graph_entry_get(&graph_entry.address()).await?; + assert_eq!(txs, graph_entry.clone()); + println!("graph_entry got 1"); + + // try put another graph_entry with the same address + let content2 = [1u8; 32]; + let graph_entry2 = GraphEntry::new(&key, vec![], content2, vec![]); + let payment_option = PaymentOption::from(&wallet); + let res = client + .graph_entry_put(graph_entry2.clone(), payment_option) + .await; + + assert!(matches!( + res, + Err(GraphError::AlreadyExists(address)) + if address == graph_entry2.address() + )); + + // try to put a graph entry linking to the first graph entry + let key3 = bls::SecretKey::random(); + let graph_entry3 = GraphEntry::new(&key3, vec![graph_entry.owner], content2, vec![]); + let payment_option = PaymentOption::from(&wallet); + client + .graph_entry_put(graph_entry3.clone(), payment_option) + .await?; + + // wait for the graph_entry to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the graph_entry is stored + let txs = client.graph_entry_get(&graph_entry3.address()).await?; + assert_eq!(txs, graph_entry3.clone()); + println!("graph_entry got 2"); + + Ok(()) +} diff --git a/autonomi/tests/pointer.rs b/autonomi/tests/pointer.rs new file mode 100644 index 0000000000..b354f2506e --- /dev/null +++ b/autonomi/tests/pointer.rs @@ -0,0 +1,118 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_logging::LogBuilder; +use autonomi::client::payment::PaymentOption; +use autonomi::AttoTokens; +use autonomi::{ + chunk::ChunkAddress, + client::pointer::{Pointer, PointerTarget}, + Client, +}; +use eyre::Result; +use serial_test::serial; +use test_utils::evm::get_funded_wallet; +use xor_name::XorName; + +#[tokio::test] +#[serial] +async fn pointer_put_manual() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("pointer", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let public_key = key.public_key(); + let target = + PointerTarget::ChunkAddress(ChunkAddress::new(XorName::random(&mut rand::thread_rng()))); + let pointer = Pointer::new(&key, 0, target); + + // estimate the cost of the pointer + let cost = client.pointer_cost(&public_key).await?; + println!("pointer cost: {cost}"); + + // put the pointer + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client.pointer_put(pointer.clone(), payment_option).await?; + assert_eq!(addr, pointer.address()); + println!("pointer put 1 cost: {cost}"); + + // wait for the pointer to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the pointer is stored + let got = client.pointer_get(&addr).await?; + assert_eq!(got, pointer.clone()); + println!("pointer got 1"); + + // try update pointer and make it point to itself + let target2 = PointerTarget::PointerAddress(addr); + let pointer2 = Pointer::new(&key, 1, target2); + let payment_option = PaymentOption::from(&wallet); + let (cost, _) = client.pointer_put(pointer2.clone(), payment_option).await?; + assert_eq!(cost, AttoTokens::zero()); + println!("pointer put 2 cost: {cost}"); + + // wait for the pointer to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the pointer is updated + let got = client.pointer_get(&addr).await?; + assert_eq!(got, pointer2.clone()); + println!("pointer got 2"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn pointer_put() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("pointer", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let public_key = key.public_key(); + let target = + PointerTarget::ChunkAddress(ChunkAddress::new(XorName::random(&mut rand::thread_rng()))); + + // estimate the cost of the pointer + let cost = client.pointer_cost(&public_key).await?; + println!("pointer cost: {cost}"); + + // put the pointer + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client + .pointer_create(&key, target.clone(), payment_option) + .await?; + println!("pointer create cost: {cost}"); + + // wait for the pointer to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the pointer is stored + let got = client.pointer_get(&addr).await?; + assert_eq!(got, Pointer::new(&key, 0, target)); + println!("pointer got 1"); + + // try update pointer and make it point to itself + let target2 = PointerTarget::PointerAddress(addr); + client.pointer_update(&key, target2.clone()).await?; + + // wait for the pointer to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the pointer is updated + let got = client.pointer_get(&addr).await?; + assert_eq!(got, Pointer::new(&key, 1, target2)); + println!("pointer got 2"); + + Ok(()) +} diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index df9a9fbce8..570ab06658 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -19,9 +19,9 @@ async fn put() -> Result<()> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put_public(data.clone(), wallet.into()).await?; + let (_cost, addr) = client.data_put_public(data.clone(), wallet.into()).await?; - let data_fetched = client.data_get_public(addr).await?; + let data_fetched = client.data_get_public(&addr).await?; assert_eq!(data, data_fetched, "data fetched should match data put"); Ok(()) diff --git a/autonomi/tests/python/conftest.py b/autonomi/tests/python/conftest.py new file mode 100644 index 0000000000..cbbd1c6f2b --- /dev/null +++ b/autonomi/tests/python/conftest.py @@ -0,0 +1,5 @@ +import os +import sys + +# Add the project root to Python path +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) \ No newline at end of file diff --git a/autonomi/tests/python/test_bindings.py b/autonomi/tests/python/test_bindings.py new file mode 100644 index 0000000000..ce1d37cd10 --- /dev/null +++ b/autonomi/tests/python/test_bindings.py @@ -0,0 +1,92 @@ +import pytest +from autonomi_client import ( + ChunkAddress, + PointerTarget, + Pointer, + PointerAddress, + SecretKey, + PublicKey, + Wallet +) + +def test_chunk_address_creation(): + # Test creating a ChunkAddress from bytes + test_data = b"test data for chunk address" + chunk_addr = ChunkAddress(test_data) + + # Test hex representation + hex_str = chunk_addr.hex + assert isinstance(hex_str, str) + assert len(hex_str) == 64 # 32 bytes = 64 hex chars + + # Test string representation + str_repr = str(chunk_addr) + assert str_repr == hex_str + + # Test repr + repr_str = repr(chunk_addr) + assert repr_str == f"ChunkAddress({hex_str})" + +def test_chunk_address_from_hex(): + # Create a chunk address + original = ChunkAddress(b"test data") + hex_str = original.hex + + # Create new chunk address from hex + recreated = ChunkAddress.from_chunk_address(hex_str) + assert recreated.hex == hex_str + +def test_pointer_target_with_chunk_address(): + # Create a chunk address + chunk_addr = ChunkAddress(b"test data for pointer target") + + # Create pointer target from chunk address + target = PointerTarget.from_chunk_address(chunk_addr) + + # Verify the hex matches + assert isinstance(target.hex, str) + assert len(target.hex) == 64 + +def test_pointer_creation(): + # Create necessary components + owner = PublicKey() + counter = 42 + chunk_addr = ChunkAddress(b"test data for pointer") + target = PointerTarget.from_chunk_address(chunk_addr) + key = SecretKey() + + # Create pointer + pointer = Pointer(owner, counter, target, key) + + # Verify pointer properties + assert isinstance(pointer.hex, str) + assert len(pointer.hex) == 64 + + # Test network address + addr = pointer.network_address() + assert isinstance(addr, PointerAddress) + assert isinstance(addr.hex, str) + assert len(addr.hex) == 64 + +def test_pointer_target_creation(): + # Test direct creation + test_data = b"test data for pointer target" + target = PointerTarget(test_data) + + # Verify hex + assert isinstance(target.hex, str) + assert len(target.hex) == 64 + + # Test from_xorname + target2 = PointerTarget.from_xorname(test_data) + assert isinstance(target2.hex, str) + assert len(target2.hex) == 64 + +def test_invalid_hex(): + # Test invalid hex string for chunk address + with pytest.raises(ValueError): + ChunkAddress.from_chunk_address("invalid hex") + + # Test invalid hex string for pointer address + with pytest.raises(ValueError): + PointerAddress("invalid hex") \ No newline at end of file diff --git a/autonomi/tests/register.rs b/autonomi/tests/register.rs deleted file mode 100644 index 0709779d5c..0000000000 --- a/autonomi/tests/register.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -#![cfg(feature = "registers")] -#![allow(deprecated)] - -use ant_logging::LogBuilder; -use autonomi::Client; -use bytes::Bytes; -use eyre::Result; -use rand::Rng; -use std::time::Duration; -use test_utils::evm::get_funded_wallet; -use tokio::time::sleep; - -#[tokio::test] -async fn register() -> Result<()> { - let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("register", false); - - let client = Client::init_local().await?; - let wallet = get_funded_wallet(); - - // Owner key of the register. - let key = bls::SecretKey::random(); - - // Create a register with the value [1, 2, 3, 4] - let rand_name: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(10) - .map(char::from) - .collect(); - let register = client - .register_create( - Some(vec![1, 2, 3, 4].into()), - &rand_name, - key.clone(), - &wallet, - ) - .await - .unwrap(); - - sleep(Duration::from_secs(10)).await; - - // Fetch the register again - let register = client.register_get(*register.address()).await.unwrap(); - - // Update the register with the value [5, 6, 7, 8] - client - .register_update(register.clone(), vec![5, 6, 7, 8].into(), key) - .await - .unwrap(); - - sleep(Duration::from_secs(2)).await; - - // Fetch and verify the register contains the updated value - let register = client.register_get(*register.address()).await.unwrap(); - assert_eq!(register.values(), vec![Bytes::from(vec![5, 6, 7, 8])]); - - Ok(()) -} diff --git a/autonomi/tests/registers.rs b/autonomi/tests/registers.rs new file mode 100644 index 0000000000..29690e3af2 --- /dev/null +++ b/autonomi/tests/registers.rs @@ -0,0 +1,164 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_logging::LogBuilder; +use autonomi::{ + client::{payment::PaymentOption, register::RegisterAddress}, + graph::GraphError, + register::RegisterError, + Client, +}; +use eyre::Result; +use serial_test::serial; +use test_utils::evm::get_funded_wallet; + +#[tokio::test] +#[serial] +async fn registers_usage() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("registers", false); + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + let main_key = bls::SecretKey::random(); + + let register_key = Client::register_key_from_name(&main_key, "register1"); + let content = Client::register_value_from_bytes(b"Hello, World!")?; + let cost = client.register_cost(®ister_key.public_key()).await?; + println!("register cost: {cost}"); + + // create the register + let (cost, addr) = client + .register_create(®ister_key, content, PaymentOption::from(&wallet)) + .await?; + println!("register created: {cost} {addr:?}"); + assert_eq!(addr, RegisterAddress::new(register_key.public_key())); + + // wait for the register to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // get the register + let value = client.register_get(&addr).await?; + assert_eq!(value, content); + + // update the register + let new_content = Client::register_value_from_bytes(b"any 32 bytes of fresh data")?; + let cost = client + .register_update(®ister_key, new_content, PaymentOption::from(&wallet)) + .await?; + println!("register updated: {cost}"); + + // wait for the register to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // get the register again + let value = client.register_get(&addr).await?; + assert_eq!(value, new_content); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn registers_errors() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("registers2", false); + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + let main_key = bls::SecretKey::random(); + + let register_key = Client::register_key_from_name(&main_key, "register1"); + let content = Client::register_value_from_bytes(b"Hello, World!")?; + let cost = client.register_cost(®ister_key.public_key()).await?; + println!("register cost: {cost}"); + + // try to update non existing register + let res = client + .register_update(®ister_key, content, PaymentOption::from(&wallet)) + .await; + println!("register update without creating should fail: {res:?}"); + assert!(matches!( + res.unwrap_err(), + RegisterError::CannotUpdateNewRegister + )); + + // create the register + let (cost, addr) = client + .register_create(®ister_key, content, PaymentOption::from(&wallet)) + .await?; + println!("register created: {cost} {addr:?}"); + assert_eq!(addr, RegisterAddress::new(register_key.public_key())); + + // wait for the register to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // try to create the register again + let res = client + .register_create(®ister_key, content, PaymentOption::from(&wallet)) + .await; + println!("register create second time should fail: {res:?}"); + assert!(matches!( + res.unwrap_err(), + RegisterError::GraphError(GraphError::AlreadyExists(_)) + )); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_register_history() -> Result<()> { + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + let main_key = bls::SecretKey::random(); + let register_key = Client::register_key_from_name(&main_key, "history_test"); + let content1 = Client::register_value_from_bytes(b"Massive")?; + let (_cost, addr) = client + .register_create(®ister_key, content1, PaymentOption::from(&wallet)) + .await?; + + // let the network replicate the register + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + let mut history = client.register_history(&addr); + let first = history.next().await?; + assert_eq!(first, Some(content1)); + let second = history.next().await?; + assert_eq!(second, None); + + let content2 = Client::register_value_from_bytes(b"Array")?; + client + .register_update(®ister_key, content2, PaymentOption::from(&wallet)) + .await?; + + // let the network replicate the updates + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + let all = client.register_history(&addr).collect().await?; + assert_eq!(all.len(), 2); + assert_eq!(all[0], content1); + assert_eq!(all[1], content2); + + let content3 = Client::register_value_from_bytes(b"Internet")?; + client + .register_update(®ister_key, content3, PaymentOption::from(&wallet)) + .await?; + let content4 = Client::register_value_from_bytes(b"Disk")?; + client + .register_update(®ister_key, content4, PaymentOption::from(&wallet)) + .await?; + + // let the network replicate the updates + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + let all = client.register_history(&addr).collect().await?; + assert_eq!(all.len(), 4); + assert_eq!(all[0], content1); + assert_eq!(all[1], content2); + assert_eq!(all[2], content3); + assert_eq!(all[3], content4); + + Ok(()) +} diff --git a/autonomi/tests/scratchpad.rs b/autonomi/tests/scratchpad.rs new file mode 100644 index 0000000000..bf3f4bbc78 --- /dev/null +++ b/autonomi/tests/scratchpad.rs @@ -0,0 +1,213 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_logging::LogBuilder; +use autonomi::client::payment::PaymentOption; +use autonomi::scratchpad::ScratchpadError; +use autonomi::AttoTokens; +use autonomi::{ + client::scratchpad::{Bytes, Scratchpad}, + Client, +}; +use eyre::Result; +use serial_test::serial; +use test_utils::evm::get_funded_wallet; + +#[tokio::test] +#[serial] +async fn scratchpad_put_manual() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("scratchpad", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let public_key = key.public_key(); + let content = Bytes::from("Massive Array of Internet Disks"); + let scratchpad = Scratchpad::new(&key, 42, &content, 0); + + // estimate the cost of the scratchpad + let cost = client.scratchpad_cost(&public_key).await?; + println!("scratchpad cost: {cost}"); + + // put the scratchpad + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client + .scratchpad_put(scratchpad.clone(), payment_option) + .await?; + assert_eq!(addr, *scratchpad.address()); + println!("scratchpad put 1 cost: {cost}"); + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is stored + let got = client.scratchpad_get(&addr).await?; + assert_eq!(got, scratchpad.clone()); + println!("scratchpad got 1"); + + // check that the content is decrypted correctly + let got_content = got.decrypt_data(&key)?; + assert_eq!(got_content, content); + + // try update scratchpad + let content2 = Bytes::from("Secure Access For Everyone"); + let scratchpad2 = Scratchpad::new(&key, 42, &content2, 1); + let payment_option = PaymentOption::from(&wallet); + let (cost, _) = client + .scratchpad_put(scratchpad2.clone(), payment_option) + .await?; + assert_eq!(cost, AttoTokens::zero()); + println!("scratchpad put 2 cost: {cost}"); + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is updated + let got = client.scratchpad_get(&addr).await?; + assert_eq!(got, scratchpad2.clone()); + println!("scratchpad got 2"); + + // check that the content is decrypted correctly + let got_content2 = got.decrypt_data(&key)?; + assert_eq!(got_content2, content2); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn scratchpad_put() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("scratchpad", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let public_key = key.public_key(); + let content = Bytes::from("what's the meaning of life the universe and everything?"); + let content_type = 42; + + // estimate the cost of the scratchpad + let cost = client.scratchpad_cost(&public_key).await?; + println!("scratchpad cost: {cost}"); + + // put the scratchpad + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client + .scratchpad_create(&key, content_type, &content, payment_option) + .await?; + println!("scratchpad create cost: {cost}"); + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is stored + let got = client.scratchpad_get(&addr).await?; + assert_eq!(*got.owner(), public_key); + assert_eq!(got.data_encoding(), content_type); + assert_eq!(got.decrypt_data(&key), Ok(content.clone())); + assert_eq!(got.counter(), 0); + assert!(got.verify_signature()); + println!("scratchpad got 1"); + + // check that the content is decrypted correctly + let got_content = got.decrypt_data(&key)?; + assert_eq!(got_content, content); + + // try update scratchpad + let content2 = Bytes::from("42"); + client + .scratchpad_update(&key, content_type, &content2) + .await?; + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is updated + let got = client.scratchpad_get(&addr).await?; + assert_eq!(*got.owner(), public_key); + assert_eq!(got.data_encoding(), content_type); + assert_eq!(got.decrypt_data(&key), Ok(content2.clone())); + assert_eq!(got.counter(), 1); + assert!(got.verify_signature()); + println!("scratchpad got 2"); + + // check that the content is decrypted correctly + let got_content2 = got.decrypt_data(&key)?; + assert_eq!(got_content2, content2); + Ok(()) +} + +#[tokio::test] +#[serial] +async fn scratchpad_errors() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("scratchpad", false); + + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + + let key = bls::SecretKey::random(); + let content = Bytes::from("what's the meaning of life the universe and everything?"); + let content_type = 42; + + // try update scratchpad, it should fail as we haven't created it + let res = client.scratchpad_update(&key, content_type, &content).await; + assert!(matches!( + res, + Err(ScratchpadError::CannotUpdateNewScratchpad) + )); + + // put the scratchpad normally + let payment_option = PaymentOption::from(&wallet); + let (cost, addr) = client + .scratchpad_create(&key, content_type, &content, payment_option) + .await?; + println!("scratchpad create cost: {cost}"); + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is stored + let got = client.scratchpad_get(&addr).await?; + assert_eq!(*got.owner(), key.public_key()); + assert_eq!(got.data_encoding(), content_type); + assert_eq!(got.decrypt_data(&key), Ok(content.clone())); + assert_eq!(got.counter(), 0); + assert!(got.verify_signature()); + println!("scratchpad got 1"); + + // try create scratchpad at the same address + let fork_content = Bytes::from("Fork"); + let payment_option = PaymentOption::from(&wallet); + let res = client + .scratchpad_create(&key, content_type, &fork_content, payment_option) + .await; + println!("Scratchpad create should fail here: {res:?}"); + assert!(matches!( + res, + Err(ScratchpadError::ScratchpadAlreadyExists(_)) + )); + + // wait for the scratchpad to be replicated + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + // check that the scratchpad is stored with original content + let got = client.scratchpad_get(&addr).await?; + assert_eq!(*got.owner(), key.public_key()); + assert_eq!(got.data_encoding(), content_type); + assert_eq!(got.decrypt_data(&key), Ok(content.clone())); + assert_eq!(got.counter(), 0); + assert!(got.verify_signature()); + println!("scratchpad got 1"); + + // check that the content is decrypted correctly and matches the original + let got_content = got.decrypt_data(&key)?; + assert_eq!(got_content, content); + Ok(()) +} diff --git a/autonomi/tests/transaction.rs b/autonomi/tests/transaction.rs deleted file mode 100644 index b0523618b3..0000000000 --- a/autonomi/tests/transaction.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use ant_logging::LogBuilder; -use ant_protocol::storage::Transaction; -use autonomi::{client::transactions::TransactionError, Client}; -use eyre::Result; -use test_utils::evm::get_funded_wallet; - -#[tokio::test] -async fn transaction_put() -> Result<()> { - let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("transaction", false); - - let client = Client::init_local().await?; - let wallet = get_funded_wallet(); - - let key = bls::SecretKey::random(); - let content = [0u8; 32]; - let transaction = Transaction::new(key.public_key(), vec![], content, vec![], &key); - - // estimate the cost of the transaction - let cost = client.transaction_cost(key.clone()).await?; - println!("transaction cost: {cost}"); - - // put the transaction - client.transaction_put(transaction.clone(), &wallet).await?; - println!("transaction put 1"); - - // wait for the transaction to be replicated - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - - // check that the transaction is stored - let txs = client.transaction_get(transaction.address()).await?; - assert_eq!(txs, vec![transaction.clone()]); - println!("transaction got 1"); - - // try put another transaction with the same address - let content2 = [1u8; 32]; - let transaction2 = Transaction::new(key.public_key(), vec![], content2, vec![], &key); - let res = client.transaction_put(transaction2.clone(), &wallet).await; - - assert!(matches!( - res, - Err(TransactionError::TransactionAlreadyExists(address)) - if address == transaction2.address() - )); - Ok(()) -} diff --git a/autonomi/tests/vault.rs b/autonomi/tests/vault.rs new file mode 100644 index 0000000000..4cf06b3677 --- /dev/null +++ b/autonomi/tests/vault.rs @@ -0,0 +1,100 @@ +// Copyright 2025 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use ant_evm::AttoTokens; +use ant_logging::LogBuilder; +use autonomi::{vault::app_name_to_vault_content_type, Client}; +use eyre::Result; +use serial_test::serial; +use test_utils::{evm::get_funded_wallet, gen_random_data}; + +#[tokio::test] +#[serial] +async fn vault_cost() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("vault", false); + let client = Client::init_local().await?; + let main_key = bls::SecretKey::random(); + + // Quoting cost for a Vault with 1TB max_size + let cost = client + .vault_cost(&main_key, 1024 * 1024 * 1024 * 1024) + .await?; + println!("1TB Vault cost: {cost}"); + + assert_eq!(cost, AttoTokens::from_u64(787416)); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn vault_expand() -> Result<()> { + let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("vault", false); + let client = Client::init_local().await?; + let wallet = get_funded_wallet(); + let main_key = bls::SecretKey::random(); + + let content_type = app_name_to_vault_content_type("TestData"); + let original_content = gen_random_data(1024); + + let cost = client + .write_bytes_to_vault( + original_content.clone(), + wallet.clone().into(), + &main_key, + content_type, + ) + .await?; + println!("1KB Vault update cost: {cost}"); + + let (fetched_content, fetched_content_type) = client.fetch_and_decrypt_vault(&main_key).await?; + println!("1KB Vault fetched"); + assert_eq!(fetched_content_type, content_type); + assert_eq!(fetched_content, original_content); + + // Update content to 2KB. Shall not incur any cost. + let update_content_2_kb = gen_random_data(2 * 1024); + let cost = client + .write_bytes_to_vault( + update_content_2_kb.clone(), + wallet.clone().into(), + &main_key, + content_type, + ) + .await?; + assert_eq!(cost, AttoTokens::zero()); + println!("2KB Vault update cost: {cost}"); + + let (fetched_content, fetched_content_type) = client.fetch_and_decrypt_vault(&main_key).await?; + println!("2KB Vault fetched"); + assert_eq!(fetched_content_type, content_type); + assert_eq!(fetched_content, update_content_2_kb); + + // Update content to 10MB. Shall only incur cost paying two extra Scratchpad. + let update_content_10_mb = gen_random_data(10 * 1024 * 1024); + let cost = client + .write_bytes_to_vault( + update_content_10_mb.clone(), + wallet.into(), + &main_key, + content_type, + ) + .await?; + assert_eq!(cost, AttoTokens::from_u64(6)); + println!("10MB Vault update cost: {cost}"); + + // Short break is required to avoid client choked by the last query round + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + + let (fetched_content, fetched_content_type) = client.fetch_and_decrypt_vault(&main_key).await?; + println!("10MB Vault fetched"); + assert_eq!(fetched_content_type, content_type); + assert_eq!(fetched_content, update_content_10_mb); + + Ok(()) +} diff --git a/autonomi/tests/wallet.rs b/autonomi/tests/wallet.rs index 33880ca5ab..54cff7150c 100644 --- a/autonomi/tests/wallet.rs +++ b/autonomi/tests/wallet.rs @@ -6,10 +6,10 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_evm::get_evm_network_from_env; -use ant_evm::EvmWallet; -use ant_evm::{Amount, RewardsAddress}; use ant_logging::LogBuilder; +use autonomi::get_evm_network; +use autonomi::Wallet; +use autonomi::{Amount, RewardsAddress}; use const_hex::traits::FromHex; use test_utils::evm::get_funded_wallet; @@ -17,8 +17,8 @@ use test_utils::evm::get_funded_wallet; async fn from_private_key() { let private_key = "0xdb1049e76a813c94be0df47ec3e20533ca676b1b9fef2ddbce9daa117e4da4aa"; let network = - get_evm_network_from_env().expect("Could not get EVM network from environment variables"); - let wallet = EvmWallet::new_from_private_key(network, private_key).unwrap(); + get_evm_network(true).expect("Could not get EVM network from environment variables"); + let wallet = Wallet::new_from_private_key(network, private_key).unwrap(); assert_eq!( wallet.address(), @@ -31,10 +31,10 @@ async fn send_tokens() { let _log_appender_guard = LogBuilder::init_single_threaded_tokio_test("wallet", false); let network = - get_evm_network_from_env().expect("Could not get EVM network from environment variables"); + get_evm_network(true).expect("Could not get EVM network from environment variables"); let wallet = get_funded_wallet(); - let receiving_wallet = EvmWallet::new_with_random_wallet(network); + let receiving_wallet = Wallet::new_with_random_wallet(network); let initial_balance = receiving_wallet.balance_of_tokens().await.unwrap(); diff --git a/autonomi/tests/wasm.rs b/autonomi/tests/wasm.rs deleted file mode 100644 index d0531c0999..0000000000 --- a/autonomi/tests/wasm.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -#![cfg(target_arch = "wasm32")] - -use std::time::Duration; - -use ant_networking::target_arch::sleep; -use autonomi::Client; -use test_utils::{evm::get_funded_wallet, gen_random_data}; -use wasm_bindgen_test::*; - -wasm_bindgen_test_configure!(run_in_browser); - -#[wasm_bindgen_test] -async fn put() -> Result<(), Box> { - enable_logging_wasm("ant-networking,autonomi,wasm"); - - let client = Client::init_local().await?; - let wallet = get_funded_wallet(); - let data = gen_random_data(1024 * 1024 * 10); - - let addr = client.data_put_public(data.clone(), wallet.into()).await?; - - sleep(Duration::from_secs(10)).await; - - let data_fetched = client.data_get_public(addr).await?; - assert_eq!(data, data_fetched, "data fetched should match data put"); - - Ok(()) -} - -fn enable_logging_wasm(directive: impl AsRef) { - use tracing_subscriber::prelude::*; - - console_error_panic_hook::set_once(); - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_ansi(false) // Only partially supported across browsers - .without_time() // std::time is not available in browsers - .with_writer(tracing_web::MakeWebConsoleWriter::new()); // write events to the console - tracing_subscriber::registry() - .with(fmt_layer) - .with(tracing_subscriber::EnvFilter::new(directive)) - .init(); -} diff --git a/docs/online-documentation/api/ant-node/README.md b/docs/online-documentation/api/ant-node/README.md new file mode 100644 index 0000000000..5f827e2673 --- /dev/null +++ b/docs/online-documentation/api/ant-node/README.md @@ -0,0 +1,259 @@ +# Ant Node API Reference + +The Ant Node provides a comprehensive API for running and managing nodes in the Autonomi network. This documentation covers both the Python bindings and the Rust implementation. + +## Installation + +=== "Python" + ```bash + # Install using uv (recommended) + curl -LsSf | sh + uv pip install maturin + uv pip install antnode + + # Or using pip + pip install antnode + ``` + +=== "Rust" + ```toml + # Add to Cargo.toml + [dependencies] + ant-node = "0.3.2" + ``` + +## Basic Usage + +=== "Python" + ```python + from antnode import AntNode + + # Create and start a node + node = AntNode() + node.run( + rewards_address="0x1234567890123456789012345678901234567890", # Your EVM wallet address + evm_network="arbitrum_sepolia", # or "arbitrum_one" for mainnet + ip="0.0.0.0", + port=12000, + initial_peers=[ + "/ip4/142.93.37.4/udp/40184/quic-v1/p2p/12D3KooWPC8q7QGZsmuTtCYxZ2s3FPXPZcS8LVKkayXkVFkqDEQB", + ], + local=False, + root_dir=None, # Uses default directory + home_network=False + ) + ``` + +=== "Rust" + ```rust + use ant_node::{NodeBuilder, NodeEvent}; + use ant_evm::RewardsAddress; + use libp2p::Multiaddr; + + // Create and start a node + let node = NodeBuilder::new() + .rewards_address(rewards_address) + .evm_network(evm_network) + .ip(ip) + .port(port) + .initial_peers(initial_peers) + .local(false) + .root_dir(None) + .home_network(false) + .build()?; + ``` + +## Core Features + +### Node Information + +=== "Python" + ```python + # Get node's peer ID + peer_id = node.peer_id() + + # Get current rewards address + address = node.get_rewards_address() + + # Get routing table information + kbuckets = node.get_kbuckets() + for distance, peers in kbuckets: + print(f"Distance {distance}: {len(peers)} peers") + + # Get all stored record addresses + records = node.get_all_record_addresses() + ``` + +=== "Rust" + ```rust + // Get node's peer ID + let peer_id = node.peer_id(); + + // Get current rewards address + let address = node.rewards_address(); + + // Get routing table information + let kbuckets = node.get_kbuckets()?; + for (distance, peers) in kbuckets { + println!("Distance {}: {} peers", distance, peers.len()); + } + + // Get all stored record addresses + let records = node.get_all_record_addresses()?; + ``` + +### Storage Operations + +=== "Python" + ```python + # Store data + key = "0123456789abcdef" # Hex string + value = b"Hello, World!" + node.store_record(key, value, "chunk") + + # Retrieve data + data = node.get_record(key) + + # Delete data + success = node.delete_record(key) + + # Get total storage size + size = node.get_stored_records_size() + ``` + +=== "Rust" + ```rust + use ant_protocol::storage::ValidationType; + + // Store data + let key = "0123456789abcdef"; // Hex string + let value = b"Hello, World!"; + node.store_record(key, value, ValidationType::Chunk)?; + + // Retrieve data + let data = node.get_record(key)?; + + // Delete data + let success = node.delete_record(key)?; + + // Get total storage size + let size = node.get_stored_records_size()?; + ``` + +### Directory Management + +=== "Python" + ```python + # Get various directory paths + root_dir = node.get_root_dir() + logs_dir = node.get_logs_dir() + data_dir = node.get_data_dir() + + # Get default directory for a specific peer + default_dir = AntNode.get_default_root_dir(peer_id) + ``` + +=== "Rust" + ```rust + // Get various directory paths + let root_dir = node.root_dir(); + let logs_dir = node.logs_dir(); + let data_dir = node.data_dir(); + + // Get default directory for a specific peer + let default_dir = Node::get_default_root_dir(peer_id)?; + ``` + +## Event Handling + +=== "Python" + ```python + # Event handling is automatic in Python bindings + # Events are logged and can be monitored through the logging system + ``` + +=== "Rust" + ```rust + use ant_node::{NodeEvent, NodeEventsReceiver}; + + // Get event receiver + let mut events: NodeEventsReceiver = node.event_receiver(); + + // Handle events + while let Ok(event) = events.recv().await { + match event { + NodeEvent::ConnectedToNetwork => println!("Connected to network"), + NodeEvent::ChunkStored(addr) => println!("Chunk stored: {}", addr), + NodeEvent::RewardReceived(amount, addr) => { + println!("Reward received: {} at {}", amount, addr) + } + NodeEvent::ChannelClosed => break, + NodeEvent::TerminateNode(reason) => { + println!("Node terminated: {}", reason); + break; + } + } + } + ``` + +## Configuration Options + +### Node Configuration + +- `rewards_address`: EVM wallet address for receiving rewards +- `evm_network`: Network to use ("arbitrum_sepolia" or "arbitrum_one") +- `ip`: IP address to listen on +- `port`: Port to listen on +- `initial_peers`: List of initial peers to connect to +- `local`: Whether to run in local mode +- `root_dir`: Custom root directory path +- `home_network`: Whether the node is behind NAT + +### Network Types + +- `arbitrum_sepolia`: Test network +- `arbitrum_one`: Main network + +## Error Handling + +=== "Python" + ```python + try: + node.store_record(key, value, "chunk") + except Exception as e: + print(f"Error storing record: {e}") + ``` + +=== "Rust" + ```rust + use ant_node::error::Error; + + match node.store_record(key, value, ValidationType::Chunk) { + Ok(_) => println!("Record stored successfully"), + Err(Error::StorageFull) => println!("Storage is full"), + Err(Error::InvalidKey) => println!("Invalid key format"), + Err(e) => println!("Other error: {}", e), + } + ``` + +## Best Practices + +1. **Error Handling** + - Always handle potential errors appropriately + - Implement retry logic for network operations + - Log errors for debugging + +2. **Resource Management** + - Monitor storage usage + - Clean up unused records + - Handle events promptly + +3. **Network Operations** + - Use appropriate timeouts + - Handle network disconnections + - Maintain peer connections + +4. **Security** + - Validate input data + - Secure storage of keys + - Regular backups of important data diff --git a/docs/online-documentation/api/ant-node/configuration.md b/docs/online-documentation/api/ant-node/configuration.md new file mode 100644 index 0000000000..c4d457723c --- /dev/null +++ b/docs/online-documentation/api/ant-node/configuration.md @@ -0,0 +1,201 @@ +# Node Configuration + +This page documents the configuration options for running an Ant Node. + +## Configuration Options + +### Network Configuration + +=== "Python" + ```python + from antnode import NodeConfig + + config = NodeConfig( + # Network settings + ip="0.0.0.0", # IP address to listen on + port=12000, # Port to listen on + evm_network="arbitrum_sepolia", # EVM network to use + rewards_address="0x...", # EVM wallet address for rewards + + # Node settings + local=False, # Run in local mode + home_network=False, # Node is behind NAT + root_dir=None, # Custom root directory + + # Network peers + initial_peers=[ # Bootstrap peers + "/ip4/142.93.37.4/udp/40184/quic-v1/p2p/12D3KooWPC8q7QGZsmuTtCYxZ2s3FPXPZcS8LVKkayXkVFkqDEQB", + ] + ) + ``` + +=== "Rust" + ```rust + use ant_node::{NodeConfig, RewardsAddress}; + use std::path::PathBuf; + + let config = NodeConfig::builder() + // Network settings + .ip("0.0.0.0") + .port(12000) + .evm_network("arbitrum_sepolia") + .rewards_address(RewardsAddress::new("0x...")) + + // Node settings + .local(false) + .home_network(false) + .root_dir(Some(PathBuf::from("/path/to/data"))) + + // Network peers + .initial_peers(vec![ + "/ip4/142.93.37.4/udp/40184/quic-v1/p2p/12D3KooWPC8q7QGZsmuTtCYxZ2s3FPXPZcS8LVKkayXkVFkqDEQB" + .parse() + .unwrap() + ]) + .build()?; + ``` + +### Storage Configuration + +=== "Python" + ```python + from antnode import StorageConfig + + storage_config = StorageConfig( + max_size=1024 * 1024 * 1024, # 1GB max storage + min_free_space=1024 * 1024, # 1MB min free space + cleanup_interval=3600, # Cleanup every hour + backup_enabled=True, + backup_interval=86400, # Daily backups + backup_path="/path/to/backups" + ) + + config.storage = storage_config + ``` + +=== "Rust" + ```rust + use ant_node::StorageConfig; + use std::path::PathBuf; + + let storage_config = StorageConfig::builder() + .max_size(1024 * 1024 * 1024) // 1GB max storage + .min_free_space(1024 * 1024) // 1MB min free space + .cleanup_interval(3600) // Cleanup every hour + .backup_enabled(true) + .backup_interval(86400) // Daily backups + .backup_path(PathBuf::from("/path/to/backups")) + .build()?; + + config.storage = storage_config; + ``` + +### Network Types + +The `evm_network` parameter can be one of: + +- `arbitrum_sepolia` - Test network +- `arbitrum_one` - Main network + +### Directory Structure + +The node uses the following directory structure: + +``` +root_dir/ +├── data/ # Stored data chunks +├── logs/ # Node logs +├── peers/ # Peer information +└── metadata/ # Node metadata +``` + +## Environment Variables + +The node configuration can also be set using environment variables: + +```bash +# Network settings +export ANT_NODE_IP="0.0.0.0" +export ANT_NODE_PORT="12000" +export ANT_NODE_EVM_NETWORK="arbitrum_sepolia" +export ANT_NODE_REWARDS_ADDRESS="0x..." + +# Node settings +export ANT_NODE_LOCAL="false" +export ANT_NODE_HOME_NETWORK="false" +export ANT_NODE_ROOT_DIR="/path/to/data" + +# Storage settings +export ANT_NODE_MAX_STORAGE="1073741824" # 1GB +export ANT_NODE_MIN_FREE_SPACE="1048576" # 1MB +export ANT_NODE_CLEANUP_INTERVAL="3600" +``` + +## Configuration File + +You can also provide configuration through a YAML file: + +```yaml +# config.yaml +network: + ip: "0.0.0.0" + port: 12000 + evm_network: "arbitrum_sepolia" + rewards_address: "0x..." + initial_peers: + - "/ip4/142.93.37.4/udp/40184/quic-v1/p2p/12D3KooWPC8q7QGZsmuTtCYxZ2s3FPXPZcS8LVKkayXkVFkqDEQB" + +node: + local: false + home_network: false + root_dir: "/path/to/data" + +storage: + max_size: 1073741824 # 1GB + min_free_space: 1048576 # 1MB + cleanup_interval: 3600 + backup: + enabled: true + interval: 86400 + path: "/path/to/backups" +``` + +Load the configuration file: + +=== "Python" + ```python + from antnode import load_config + + config = load_config("config.yaml") + node = AntNode(config) + ``` + +=== "Rust" + ```rust + use ant_node::config::load_config; + + let config = load_config("config.yaml")?; + let node = Node::new(config)?; + ``` + +## Best Practices + +1. **Network Settings** + - Use a static IP if possible + - Open required ports in firewall + - Configure proper rewards address + +2. **Storage Management** + - Set appropriate storage limits + - Enable regular backups + - Monitor free space + +3. **Security** + - Run node with minimal privileges + - Secure rewards address private key + - Regular security updates + +4. **Monitoring** + - Enable logging + - Monitor node health + - Set up alerts diff --git a/docs/online-documentation/api/ant-node/network.md b/docs/online-documentation/api/ant-node/network.md new file mode 100644 index 0000000000..57bfc82809 --- /dev/null +++ b/docs/online-documentation/api/ant-node/network.md @@ -0,0 +1,246 @@ +# Network Operations + +This page documents the network operations available in the Ant Node API. + +## Node Management + +### Starting a Node + +=== "Python" + ```python + from antnode import AntNode + + # Create and start a node + node = AntNode() + node.run( + rewards_address="0x1234567890123456789012345678901234567890", + evm_network="arbitrum_sepolia", + ip="0.0.0.0", + port=12000, + initial_peers=[ + "/ip4/142.93.37.4/udp/40184/quic-v1/p2p/12D3KooWPC8q7QGZsmuTtCYxZ2s3FPXPZcS8LVKkayXkVFkqDEQB", + ] + ) + ``` + +=== "Rust" + ```rust + use ant_node::{Node, NodeConfig}; + + // Create and start a node + let config = NodeConfig::default(); + let mut node = Node::new(config)?; + node.run().await?; + ``` + +### Node Information + +=== "Python" + ```python + # Get node's peer ID + peer_id = node.peer_id() + + # Get current rewards address + address = node.get_rewards_address() + + # Get routing table information + kbuckets = node.get_kbuckets() + for distance, peers in kbuckets: + print(f"Distance {distance}: {len(peers)} peers") + + # Get all stored record addresses + records = node.get_all_record_addresses() + ``` + +=== "Rust" + ```rust + // Get node's peer ID + let peer_id = node.peer_id(); + + // Get current rewards address + let address = node.rewards_address(); + + // Get routing table information + let kbuckets = node.get_kbuckets()?; + for (distance, peers) in kbuckets { + println!("Distance {}: {} peers", distance, peers.len()); + } + + // Get all stored record addresses + let records = node.get_all_record_addresses()?; + ``` + +## Network Events + +### Event Handling + +=== "Python" + ```python + from antnode import NodeEvent + + # Register event handlers + @node.on(NodeEvent.CONNECTED) + def handle_connected(): + print("Connected to network") + + @node.on(NodeEvent.CHUNK_STORED) + def handle_chunk_stored(address): + print(f"Chunk stored: {address}") + + @node.on(NodeEvent.REWARD_RECEIVED) + def handle_reward(amount, address): + print(f"Reward received: {amount} at {address}") + ``` + +=== "Rust" + ```rust + use ant_node::{NodeEvent, NodeEventsReceiver}; + + // Get event receiver + let mut events: NodeEventsReceiver = node.event_receiver(); + + // Handle events + while let Ok(event) = events.recv().await { + match event { + NodeEvent::ConnectedToNetwork => println!("Connected to network"), + NodeEvent::ChunkStored(addr) => println!("Chunk stored: {}", addr), + NodeEvent::RewardReceived(amount, addr) => { + println!("Reward received: {} at {}", amount, addr) + } + NodeEvent::ChannelClosed => break, + NodeEvent::TerminateNode(reason) => { + println!("Node terminated: {}", reason); + break; + } + } + } + ``` + +## Peer Management + +### Peer Discovery + +=== "Python" + ```python + # Add a peer manually + node.add_peer("/ip4/1.2.3.4/udp/12000/quic-v1/p2p/...") + + # Get connected peers + peers = node.get_connected_peers() + for peer in peers: + print(f"Peer: {peer.id}, Address: {peer.address}") + + # Find peers near an address + nearby = node.find_peers_near(target_address) + ``` + +=== "Rust" + ```rust + // Add a peer manually + node.add_peer("/ip4/1.2.3.4/udp/12000/quic-v1/p2p/...".parse()?)?; + + // Get connected peers + let peers = node.get_connected_peers()?; + for peer in peers { + println!("Peer: {}, Address: {}", peer.id, peer.address); + } + + // Find peers near an address + let nearby = node.find_peers_near(&target_address).await?; + ``` + +## Data Storage + +### Record Management + +=== "Python" + ```python + # Store a record + key = "0123456789abcdef" # Hex string + value = b"Hello, World!" + node.store_record(key, value, "chunk") + + # Retrieve a record + data = node.get_record(key) + + # Delete a record + success = node.delete_record(key) + + # Get total storage size + size = node.get_stored_records_size() + ``` + +=== "Rust" + ```rust + use ant_node::storage::ValidationType; + + // Store a record + let key = "0123456789abcdef"; // Hex string + let value = b"Hello, World!"; + node.store_record(key, value, ValidationType::Chunk)?; + + // Retrieve a record + let data = node.get_record(key)?; + + // Delete a record + let success = node.delete_record(key)?; + + // Get total storage size + let size = node.get_stored_records_size()?; + ``` + +## Network Metrics + +### Performance Monitoring + +=== "Python" + ```python + # Get network metrics + metrics = node.get_metrics() + print(f"Connected peers: {metrics.peer_count}") + print(f"Records stored: {metrics.record_count}") + print(f"Storage used: {metrics.storage_used}") + print(f"Bandwidth in: {metrics.bandwidth_in}") + print(f"Bandwidth out: {metrics.bandwidth_out}") + + # Get node uptime + uptime = node.get_uptime() + print(f"Node uptime: {uptime} seconds") + ``` + +=== "Rust" + ```rust + // Get network metrics + let metrics = node.get_metrics()?; + println!("Connected peers: {}", metrics.peer_count); + println!("Records stored: {}", metrics.record_count); + println!("Storage used: {}", metrics.storage_used); + println!("Bandwidth in: {}", metrics.bandwidth_in); + println!("Bandwidth out: {}", metrics.bandwidth_out); + + // Get node uptime + let uptime = node.get_uptime()?; + println!("Node uptime: {} seconds", uptime); + ``` + +## Best Practices + +1. **Event Handling** + - Always handle critical events + - Implement proper error recovery + - Log important events + +2. **Peer Management** + - Maintain healthy peer connections + - Implement peer discovery + - Handle peer disconnections + +3. **Storage Management** + - Monitor storage usage + - Implement cleanup policies + - Handle storage full conditions + +4. **Network Health** + - Monitor network metrics + - Track bandwidth usage + - Monitor node performance diff --git a/docs/online-documentation/api/autonomi-client/README.md b/docs/online-documentation/api/autonomi-client/README.md new file mode 100644 index 0000000000..d3b2f11dde --- /dev/null +++ b/docs/online-documentation/api/autonomi-client/README.md @@ -0,0 +1,640 @@ +# Autonomi API Documentation + +## Installation + +Choose your preferred language: + +=== "Node.js" + ```bash + # Note: Package not yet published to npm + # Clone the repository and build from source + git clone https://github.com/dirvine/autonomi.git + cd autonomi + npm install + ``` + +=== "Python" + ```bash + pip install autonomi + ``` + +=== "Rust" + ```toml + # Add to Cargo.toml: + [dependencies] + autonomi = "0.3.1" + ``` + +## Client Initialization + +Initialize a client in read-only mode for browsing data, or with write capabilities for full access: + +=== "Node.js" + ```typescript + import { Client } from 'autonomi'; + + // Initialize a read-only client + const client = await Client.initReadOnly(); + + // Or initialize with write capabilities and configuration + const config = { + // Add your configuration here + }; + const client = await Client.initWithConfig(config); + ``` + +=== "Python" + ```python + from autonomi import Client + + # Initialize a read-only client + client = Client.init_read_only() + + # Or initialize with write capabilities and configuration + config = { + # Add your configuration here + } + client = Client.init_with_config(config) + ``` + +=== "Rust" + ```rust + use autonomi::Client; + + // Initialize a read-only client + let client = Client::new_local().await?; + + // Or initialize with configuration + let config = ClientConfig::default(); + let client = Client::new(config).await?; + ``` + +## Core Data Types + +Autonomi provides four fundamental data types that serve as building blocks for all network operations. For detailed information about each type, see the [Data Types Guide](../../guides/data_types.md). + +### 1. Chunk + +Immutable, quantum-secure encrypted data blocks: + +=== "Node.js" + ```typescript + import { Chunk } from 'autonomi'; + + // Store raw data as a chunk + const data = Buffer.from('Hello, World!'); + const chunk = await client.storeChunk(data); + + // Retrieve chunk data + const retrieved = await client.getChunk(chunk.address); + assert(Buffer.compare(data, retrieved) === 0); + + // Get chunk metadata + const metadata = await client.getChunkMetadata(chunk.address); + console.log(`Size: ${metadata.size}`); + ``` + +=== "Python" + ```python + from autonomi import Chunk + + # Store raw data as a chunk + data = b"Hello, World!" + chunk = client.store_chunk(data) + + # Retrieve chunk data + retrieved = client.get_chunk(chunk.address) + assert data == retrieved + + # Get chunk metadata + metadata = client.get_chunk_metadata(chunk.address) + print(f"Size: {metadata.size}") + ``` + +=== "Rust" + ```rust + use autonomi::Chunk; + + // Store raw data as a chunk + let data = b"Hello, World!"; + let chunk = client.store_chunk(data).await?; + + // Retrieve chunk data + let retrieved = client.get_chunk(chunk.address()).await?; + assert_eq!(data, &retrieved[..]); + + // Get chunk metadata + let metadata = client.get_chunk_metadata(chunk.address()).await?; + println!("Size: {}", metadata.size); + ``` + +### 2. Pointer + +Mutable references with version tracking: + +=== "Node.js" + ```typescript + import { Pointer } from 'autonomi'; + + // Create a pointer to some data + const pointer = await client.createPointer(targetAddress); + + // Update pointer target + await client.updatePointer(pointer.address, newTargetAddress); + + // Resolve pointer to get current target + const target = await client.resolvePointer(pointer.address); + + // Get pointer metadata and version + const metadata = await client.getPointerMetadata(pointer.address); + console.log(`Version: ${metadata.version}`); + ``` + +=== "Python" + ```python + from autonomi import Pointer + + # Create a pointer to some data + pointer = client.create_pointer(target_address) + + # Update pointer target + client.update_pointer(pointer.address, new_target_address) + + # Resolve pointer to get current target + target = client.resolve_pointer(pointer.address) + + # Get pointer metadata and version + metadata = client.get_pointer_metadata(pointer.address) + print(f"Version: {metadata.version}") + ``` + +=== "Rust" + ```rust + use autonomi::Pointer; + + // Create a pointer to some data + let pointer = client.create_pointer(target_address).await?; + + // Update pointer target + client.update_pointer(pointer.address(), new_target_address).await?; + + // Resolve pointer to get current target + let target = client.resolve_pointer(pointer.address()).await?; + + // Get pointer metadata and version + let metadata = client.get_pointer_metadata(pointer.address()).await?; + println!("Version: {}", metadata.version); + ``` + +### 3. GraphEntry + +Decentralized Graph structures for linked data: + +=== "Node.js" + ```typescript + import { GraphEntry } from 'autonomi'; + + // Create a new graph + const entry = await client.createGraphEntry(); + + // Append items + await client.appendToGraph(entry.address, item1); + await client.appendToGraph(entry.address, item2); + + // Read graph contents + const items = await client.getGraph(entry.address); + + // Get graph history + const history = await client.getGraphHistory(entry.address); + for (const entry of history) { + console.log(`Version ${entry.version}: ${entry.data}`); + } + + // Check for forks + const forks = await client.detectForks(entry.address); + if (!forks) { + console.log('No forks detected'); + } else { + handleForks(forks.branches); + } + ``` + +=== "Python" + ```python + from autonomi import GraphEntry + + # Create a new graph + entry = client.create_graph_entry() + + # Append items + client.append_to_graph(entry.address, item1) + client.append_to_graph(entry.address, item2) + + # Read list contents + items = client.get_graph(entry.address) + + # Get graph history + history = client.get_graph_history(entry.address) + for entry in history: + print(f"Version {entry.version}: {entry.data}") + ``` + +=== "Rust" + ```rust + use autonomi::GraphEntry; + + // Create a new graph + let entry = client.create_graph_entry().await?; + + // Append items + client.append_to_graph(entry.address(), item1).await?; + client.append_to_graph(entry.address(), item2).await?; + + // Read graph contents + let items = client.get_graph(entry.address()).await?; + + // Get graph history + let history = client.get_graph_history(entry.address()).await?; + for entry in history { + println!("Version {}: {:?}", entry.version, entry.data); + } + ``` + +### 4. ScratchPad + +Unstructured data with CRDT properties: + +=== "Node.js" + ```typescript + import { ScratchPad, ContentType } from 'autonomi'; + + // Create a scratchpad + const pad = await client.createScratchpad(ContentType.UserSettings); + + // Update with data + await client.updateScratchpad(pad.address, settingsData); + + // Read current data + const current = await client.getScratchpad(pad.address); + + // Get metadata + const metadata = await client.getScratchpadMetadata(pad.address); + console.log(`Updates: ${metadata.updateCounter}`); + ``` + +=== "Python" + ```python + from autonomi import ScratchPad, ContentType + + # Create a scratchpad + pad = client.create_scratchpad(ContentType.USER_SETTINGS) + + # Update with data + client.update_scratchpad(pad.address, settings_data) + + # Read current data + current = client.get_scratchpad(pad.address) + + # Get metadata + metadata = client.get_scratchpad_metadata(pad.address) + print(f"Updates: {metadata.update_counter}") + ``` + +=== "Rust" + ```rust + use autonomi::{ScratchPad, ContentType}; + + // Create a scratchpad + let pad = client.create_scratchpad(ContentType::UserSettings).await?; + + // Update with data + client.update_scratchpad(pad.address(), settings_data).await?; + + // Read current data + let current = client.get_scratchpad(pad.address()).await?; + + // Get metadata + let metadata = client.get_scratchpad_metadata(pad.address()).await?; + println!("Updates: {}", metadata.update_counter); + ``` + +## File System Operations + +Create and manage files and directories: + +=== "Node.js" + ```typescript + import { File, Directory } from 'autonomi/fs'; + + // Store a file + const file = await client.storeFile('example.txt', content); + + // Create a directory + const dir = await client.createDirectory('docs'); + + // Add file to directory + await client.addToDirectory(dir.address, file.address); + + // List directory contents + const entries = await client.listDirectory(dir.address); + for (const entry of entries) { + if (entry.isFile) { + console.log(`File: ${entry.name}`); + } else { + console.log(`Dir: ${entry.name}`); + } + } + ``` + +=== "Python" + ```python + from autonomi.fs import File, Directory + + # Store a file + file = client.store_file("example.txt", content) + + # Create a directory + dir = client.create_directory("docs") + + # Add file to directory + client.add_to_directory(dir.address, file.address) + + # List directory contents + entries = client.list_directory(dir.address) + for entry in entries: + if entry.is_file: + print(f"File: {entry.name}") + else: + print(f"Dir: {entry.name}") + ``` + +=== "Rust" + ```rust + use autonomi::fs::{File, Directory}; + + // Store a file + let file = client.store_file("example.txt", content).await?; + + // Create a directory + let dir = client.create_directory("docs").await?; + + // Add file to directory + client.add_to_directory(dir.address(), file.address()).await?; + + // List directory contents + let entries = client.list_directory(dir.address()).await?; + for entry in entries { + match entry { + DirEntry::File(f) => println!("File: {}", f.name), + DirEntry::Directory(d) => println!("Dir: {}", d.name), + } + } + ``` + +## Error Handling + +Each language provides appropriate error handling mechanisms: + +=== "Node.js" + ```typescript + import { ChunkError, PointerError } from 'autonomi/errors'; + + // Handle chunk operations + try { + const data = await client.getChunk(address); + processData(data); + } catch (error) { + if (error instanceof ChunkError.NotFound) { + handleMissing(); + } else if (error instanceof ChunkError.NetworkError) { + handleNetworkError(error); + } else { + handleOtherError(error); + } + } + + // Handle pointer updates + try { + await client.updatePointer(address, newTarget); + console.log('Update successful'); + } catch (error) { + if (error instanceof PointerError.VersionConflict) { + handleConflict(); + } else { + handleOtherError(error); + } + } + ``` + +=== "Python" + ```python + from autonomi.errors import ChunkError, PointerError + + # Handle chunk operations + try: + data = client.get_chunk(address) + process_data(data) + except ChunkError.NotFound: + handle_missing() + except ChunkError.NetworkError as e: + handle_network_error(e) + except Exception as e: + handle_other_error(e) + + # Handle pointer updates + try: + client.update_pointer(address, new_target) + print("Update successful") + except PointerError.VersionConflict: + handle_conflict() + except Exception as e: + handle_other_error(e) + ``` + +=== "Rust" + ```rust + use autonomi::error::{ChunkError, PointerError, GraphError, ScratchPadError}; + + // Handle chunk operations + match client.get_chunk(address).await { + Ok(data) => process_data(data), + Err(ChunkError::NotFound) => handle_missing(), + Err(ChunkError::NetworkError(e)) => handle_network_error(e), + Err(e) => handle_other_error(e), + } + + // Handle pointer updates + match client.update_pointer(address, new_target).await { + Ok(_) => println!("Update successful"), + Err(PointerError::VersionConflict) => handle_conflict(), + Err(e) => handle_other_error(e), + } + ``` + +## Advanced Usage + +### Custom Types + +=== "Node.js" + ```typescript + interface MyData { + field1: string; + field2: number; + } + + // Store custom type in a scratchpad + const data: MyData = { + field1: 'test', + field2: 42 + }; + const pad = await client.createScratchpad(ContentType.Custom('MyData')); + await client.updateScratchpad(pad.address, data); + ``` + +=== "Python" + ```python + from dataclasses import dataclass + + @dataclass + class MyData: + field1: str + field2: int + + # Store custom type in a scratchpad + data = MyData(field1="test", field2=42) + pad = client.create_scratchpad(ContentType.CUSTOM("MyData")) + client.update_scratchpad(pad.address, data) + ``` + +=== "Rust" + ```rust + use serde::{Serialize, Deserialize}; + + #[derive(Serialize, Deserialize)] + struct MyData { + field1: String, + field2: u64, + } + + // Store custom type in a scratchpad + let data = MyData { + field1: "test".into(), + field2: 42, + }; + let pad = client.create_scratchpad(ContentType::Custom("MyData")).await?; + client.update_scratchpad(pad.address(), &data).await?; + ``` + +### Encryption + +=== "Node.js" + ```typescript + import { encrypt, decrypt, generateKey } from 'autonomi/crypto'; + + // Encrypt data before storage + const key = await generateAesKey(); + const encrypted = await encryptAes(data, key); + const pad = await client.createScratchpad(ContentType.Encrypted); + await client.updateScratchpad(pad.address, encrypted); + + // Decrypt retrieved data + const encrypted = await client.getScratchpad(pad.address); + const decrypted = await decryptAes(encrypted, key); + ``` + +=== "Python" + ```python + from autonomi.crypto import encrypt_aes, decrypt_aes + + # Encrypt data before storage + key = generate_aes_key() + encrypted = encrypt_aes(data, key) + pad = client.create_scratchpad(ContentType.ENCRYPTED) + client.update_scratchpad(pad.address, encrypted) + + # Decrypt retrieved data + encrypted = client.get_scratchpad(pad.address) + decrypted = decrypt_aes(encrypted, key) + ``` + +=== "Rust" + ```rust + use autonomi::crypto::{encrypt_aes, decrypt_aes}; + + // Encrypt data before storage + let key = generate_aes_key(); + let encrypted = encrypt_aes(data, &key)?; + let pad = client.create_scratchpad(ContentType::Encrypted).await?; + client.update_scratchpad(pad.address(), &encrypted).await?; + + // Decrypt retrieved data + let encrypted = client.get_scratchpad(pad.address()).await?; + let decrypted = decrypt_aes(encrypted, &key)?; + ``` + +## Best Practices + +1. **Data Type Selection** + - Use Chunks for immutable data + - Use Pointers for mutable references + - Use GraphEntrys for ordered collections + - Use ScratchPads for temporary data + +2. **Error Handling** + - Always handle network errors appropriately + - Use type-specific error handling + - Implement retry logic for transient failures + +3. **Performance** + - Use batch operations for multiple items + - Consider chunking large data sets + - Cache frequently accessed data locally + +4. **Security** + - Encrypt sensitive data before storage + - Use secure key management + - Validate data integrity + +## Type System + +=== "Node.js" + ```typescript + import { Address, Data, Metadata } from 'autonomi/types'; + + interface Client { + storeChunk(data: Buffer): Promise
; + getChunk(address: Address): Promise; + createPointer(target: Address): Promise; + updatePointer(address: Address, target: Address): Promise; + } + ``` + +=== "Python" + ```python + from typing import List, Optional, Union + from autonomi.types import Address, Data, Metadata + + class Client: + def store_chunk(self, data: bytes) -> Address: ... + def get_chunk(self, address: Address) -> bytes: ... + def create_pointer(self, target: Address) -> Pointer: ... + def update_pointer(self, address: Address, target: Address) -> None: ... + ``` + +=== "Rust" + ```rust + use autonomi::types::{Address, Data, Metadata}; + + pub trait Client { + async fn store_chunk(&self, data: &[u8]) -> Result
; + async fn get_chunk(&self, address: &Address) -> Result>; + async fn create_pointer(&self, target: Address) -> Result; + async fn update_pointer(&self, address: Address, target: Address) -> Result<()>; + } + ``` + +## Further Reading + +- [Data Types Guide](../../guides/data_types.md) +- [Client Modes Guide](../../guides/client_modes.md) +- [Local Network Setup](../../guides/local_network.md) diff --git a/docs/online-documentation/api/autonomi-client/data_types.md b/docs/online-documentation/api/autonomi-client/data_types.md new file mode 100644 index 0000000000..3b9cd4c660 --- /dev/null +++ b/docs/online-documentation/api/autonomi-client/data_types.md @@ -0,0 +1,211 @@ +# Data Types Reference + +This page provides detailed information about the core data types used in the Autonomi Client API. + +## Address + +A unique identifier for content in the network. + +=== "Node.js" + ```typescript + interface Address { + readonly bytes: Buffer; + toString(): string; + equals(other: Address): boolean; + } + ``` + +=== "Python" + ```python + class Address: + @property + def bytes(self) -> bytes: ... + def __str__(self) -> str: ... + def __eq__(self, other: Address) -> bool: ... + ``` + +=== "Rust" + ```rust + pub struct Address([u8; 32]); + + impl Address { + pub fn as_bytes(&self) -> &[u8]; + pub fn to_string(&self) -> String; + } + ``` + +## Chunk + +An immutable data block with quantum-secure encryption. + +=== "Node.js" + ```typescript + interface Chunk { + readonly address: Address; + readonly data: Buffer; + readonly size: number; + readonly type: ChunkType; + } + + enum ChunkType { + Data, + Metadata, + Index + } + ``` + +=== "Python" + ```python + class Chunk: + @property + def address(self) -> Address: ... + @property + def data(self) -> bytes: ... + @property + def size(self) -> int: ... + @property + def type(self) -> ChunkType: ... + + class ChunkType(Enum): + DATA = 1 + METADATA = 2 + INDEX = 3 + ``` + +=== "Rust" + ```rust + pub struct Chunk { + pub address: Address, + pub data: Vec, + pub size: usize, + pub type_: ChunkType, + } + + pub enum ChunkType { + Data, + Metadata, + Index, + } + ``` + +## Pointer + +A mutable reference to data in the network. + +=== "Node.js" + ```typescript + interface Pointer { + readonly address: Address; + readonly target: Address; + readonly version: number; + setTarget(target: Address): void; + } + ``` + +=== "Python" + ```python + class Pointer: + @property + def address(self) -> Address: ... + @property + def target(self) -> Address: ... + @property + def version(self) -> int: ... + def set_target(self, target: Address) -> None: ... + ``` + +=== "Rust" + ```rust + pub struct Pointer { + pub address: Address, + pub target: Address, + pub version: u64, + } + + impl Pointer { + pub fn set_target(&mut self, target: Address); + } + ``` + +## GraphEntry + +A decentralized Graph structure for linked data. + +=== "Node.js" + ```typescript + interface GraphEntry { + readonly address: Address; + readonly length: number; + append(item: T): void; + get(index: number): T; + toArray(): T[]; + } + ``` + +=== "Python" + ```python + class GraphEntry(Generic[T]): + @property + def address(self) -> Address: ... + @property + def length(self) -> int: ... + def append(self, item: T) -> None: ... + def __getitem__(self, index: int) -> T: ... + def to_list(self) -> List[T]: ... + ``` + +=== "Rust" + ```rust + pub struct GraphEntry { + pub address: Address, + pub length: usize, + } + + impl GraphEntry { + pub fn append(&mut self, item: T); + pub fn get(&self, index: usize) -> Option<&T>; + pub fn to_vec(&self) -> Vec; + } + ``` + +## ScratchPad + +Unstructured data with CRDT properties. + +=== "Node.js" + ```typescript + interface ScratchPad { + readonly address: Address; + readonly type: ContentType; + readonly updateCounter: number; + update(data: Buffer): void; + getData(): Buffer; + } + ``` + +=== "Python" + ```python + class ScratchPad: + @property + def address(self) -> Address: ... + @property + def type(self) -> ContentType: ... + @property + def update_counter(self) -> int: ... + def update(self, data: bytes) -> None: ... + def get_data(self) -> bytes: ... + ``` + +=== "Rust" + ```rust + pub struct ScratchPad { + pub address: Address, + pub type_: ContentType, + pub update_counter: u64, + } + + impl ScratchPad { + pub fn update(&mut self, data: Vec); + pub fn get_data(&self) -> Vec; + } + ``` diff --git a/docs/online-documentation/api/autonomi-client/errors.md b/docs/online-documentation/api/autonomi-client/errors.md new file mode 100644 index 0000000000..94ab492a7e --- /dev/null +++ b/docs/online-documentation/api/autonomi-client/errors.md @@ -0,0 +1,264 @@ +# Error Handling Reference + +This page documents the error types and handling patterns for the Autonomi Client API. + +## Error Types + +### ChunkError + +Errors related to chunk operations. + +=== "Node.js" + ```typescript + class ChunkError extends Error { + static NotFound: typeof ChunkError; + static InvalidSize: typeof ChunkError; + static NetworkError: typeof ChunkError; + static StorageFull: typeof ChunkError; + } + + try { + const chunk = await client.getChunk(address); + } catch (error) { + if (error instanceof ChunkError.NotFound) { + // Handle missing chunk + } else if (error instanceof ChunkError.NetworkError) { + // Handle network issues + } + } + ``` + +=== "Python" + ```python + class ChunkError(Exception): + class NotFound(ChunkError): pass + class InvalidSize(ChunkError): pass + class NetworkError(ChunkError): pass + class StorageFull(ChunkError): pass + + try: + chunk = client.get_chunk(address) + except ChunkError.NotFound: + # Handle missing chunk + pass + except ChunkError.NetworkError as e: + # Handle network issues + pass + ``` + +=== "Rust" + ```rust + pub enum ChunkError { + NotFound, + InvalidSize, + NetworkError(NetworkError), + StorageFull, + } + + match client.get_chunk(address).await { + Ok(chunk) => { /* Process chunk */ } + Err(ChunkError::NotFound) => { /* Handle missing chunk */ } + Err(ChunkError::NetworkError(e)) => { /* Handle network issues */ } + Err(e) => { /* Handle other errors */ } + } + ``` + +### PointerError + +Errors related to pointer operations. + +=== "Node.js" + ```typescript + class PointerError extends Error { + static NotFound: typeof PointerError; + static VersionConflict: typeof PointerError; + static InvalidTarget: typeof PointerError; + } + + try { + await client.updatePointer(address, newTarget); + } catch (error) { + if (error instanceof PointerError.VersionConflict) { + // Handle version conflict + } + } + ``` + +=== "Python" + ```python + class PointerError(Exception): + class NotFound(PointerError): pass + class VersionConflict(PointerError): pass + class InvalidTarget(PointerError): pass + + try: + client.update_pointer(address, new_target) + except PointerError.VersionConflict: + # Handle version conflict + pass + ``` + +=== "Rust" + ```rust + pub enum PointerError { + NotFound, + VersionConflict, + InvalidTarget, + } + + match client.update_pointer(address, new_target).await { + Ok(_) => { /* Success */ } + Err(PointerError::VersionConflict) => { /* Handle conflict */ } + Err(e) => { /* Handle other errors */ } + } + ``` + +## Error Handling Patterns + +### Retry Logic + +For transient errors like network issues: + +=== "Node.js" + ```typescript + async function withRetry( + operation: () => Promise, + maxRetries = 3, + delay = 1000 + ): Promise { + let lastError: Error; + for (let i = 0; i < maxRetries; i++) { + try { + return await operation(); + } catch (error) { + if (error instanceof ChunkError.NetworkError) { + lastError = error; + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + throw error; + } + } + throw lastError; + } + + // Usage + const chunk = await withRetry(() => client.getChunk(address)); + ``` + +=== "Python" + ```python + async def with_retry(operation, max_retries=3, delay=1.0): + last_error = None + for i in range(max_retries): + try: + return await operation() + except ChunkError.NetworkError as e: + last_error = e + await asyncio.sleep(delay) + continue + except Exception as e: + raise e + raise last_error + + # Usage + chunk = await with_retry(lambda: client.get_chunk(address)) + ``` + +=== "Rust" + ```rust + async fn with_retry( + operation: F, + max_retries: u32, + delay: Duration + ) -> Result + where + F: Fn() -> Future>, + E: From, + { + let mut last_error = None; + for_ in 0..max_retries { + match operation().await { + Ok(result) => return Ok(result), + Err(e) => { + if let Some(ChunkError::NetworkError(_)) = e.downcast_ref() { + last_error = Some(e); + tokio::time::sleep(delay).await; + continue; + } + return Err(e); + } + } + } + Err(last_error.unwrap()) + } + + // Usage + let chunk = with_retry(|| client.get_chunk(address), 3, Duration::from_secs(1)).await?; + ``` + +### Error Recovery + +For handling version conflicts in pointers: + +=== "Node.js" + ```typescript + async function updatePointerSafely( + client: Client, + address: Address, + newTarget: Address + ): Promise { + while (true) { + try { + await client.updatePointer(address, newTarget); + break; + } catch (error) { + if (error instanceof PointerError.VersionConflict) { + const current = await client.resolvePointer(address); + if (current.equals(newTarget)) break; + continue; + } + throw error; + } + } + } + ``` + +=== "Python" + ```python + async def update_pointer_safely(client, address, new_target): + while True: + try: + await client.update_pointer(address, new_target) + break + except PointerError.VersionConflict: + current = await client.resolve_pointer(address) + if current == new_target: + break + continue + except Exception as e: + raise e + ``` + +=== "Rust" + ```rust + async fn update_pointer_safely( + client: &Client, + address: Address, + new_target: Address + ) -> Result<()> { + loop { + match client.update_pointer(address, new_target).await { + Ok(_) => break Ok(()), + Err(PointerError::VersionConflict) => { + let current = client.resolve_pointer(address).await?; + if current == new_target { + break Ok(()); + } + continue; + } + Err(e) => break Err(e), + } + } + } + ``` diff --git a/docs/online-documentation/api/blsttc/README.md b/docs/online-documentation/api/blsttc/README.md new file mode 100644 index 0000000000..3f5760a133 --- /dev/null +++ b/docs/online-documentation/api/blsttc/README.md @@ -0,0 +1,258 @@ +# BLS Threshold Crypto API Reference + +BLS Threshold Crypto (blsttc) is a Rust implementation of BLS (Boneh-Lynn-Shacham) threshold signatures with support for both Rust and Python interfaces. + +## Installation + +=== "Python" + ```bash + # Install using uv (recommended) + curl -LsSf | sh + uv pip install blsttc + + # Or using pip + pip install blsttc + ``` + +=== "Rust" + ```toml + # Add to Cargo.toml + [dependencies] + blsttc = "8.0.2" + ``` + +## Basic Usage + +=== "Python" + ```python + from blsttc import SecretKey, PublicKey, Signature + + # Generate a secret key + secret_key = SecretKey.random() + + # Get the corresponding public key + public_key = secret_key.public_key() + + # Sign a message + message = b"Hello, World!" + signature = secret_key.sign(message) + + # Verify the signature + assert public_key.verify(signature, message) + ``` + +=== "Rust" + ```rust + use blsttc::{SecretKey, PublicKey, Signature}; + + // Generate a secret key + let secret_key = SecretKey::random(); + + // Get the corresponding public key + let public_key = secret_key.public_key(); + + // Sign a message + let message = b"Hello, World!"; + let signature = secret_key.sign(message); + + // Verify the signature + assert!(public_key.verify(&signature, message)); + ``` + +## Threshold Signatures + +=== "Python" + ```python + from blsttc import SecretKeySet, PublicKeySet + + # Create a threshold signature scheme + threshold = 3 # Minimum signatures required + total = 5 # Total number of shares + sk_set = SecretKeySet.random(threshold) + + # Get the public key set + pk_set = sk_set.public_keys() + + # Generate secret key shares + secret_shares = [sk_set.secret_key_share(i) for i in range(total)] + + # Sign with individual shares + message = b"Hello, World!" + sig_shares = [share.sign(message) for share in secret_shares] + + # Combine signatures + combined_sig = pk_set.combine_signatures(sig_shares[:threshold]) + + # Verify the combined signature + assert pk_set.public_key().verify(combined_sig, message) + ``` + +=== "Rust" + ```rust + use blsttc::{SecretKeySet, PublicKeySet}; + + // Create a threshold signature scheme + let threshold = 3; // Minimum signatures required + let total = 5; // Total number of shares + let sk_set = SecretKeySet::random(threshold); + + // Get the public key set + let pk_set = sk_set.public_keys(); + + // Generate secret key shares + let secret_shares: Vec<_> = (0..total) + .map(|i| sk_set.secret_key_share(i)) + .collect(); + + // Sign with individual shares + let message = b"Hello, World!"; + let sig_shares: Vec<_> = secret_shares + .iter() + .map(|share| share.sign(message)) + .collect(); + + // Combine signatures + let combined_sig = pk_set.combine_signatures(sig_shares[..threshold].iter())?; + + // Verify the combined signature + assert!(pk_set.public_key().verify(&combined_sig, message)); + ``` + +## Advanced Features + +### Key Generation + +=== "Python" + ```python + from blsttc import SecretKey, Fr + + # Generate from random seed + secret_key = SecretKey.random() + + # Generate from bytes + bytes_data = b"some-32-byte-seed" + secret_key = SecretKey.from_bytes(bytes_data) + + # Generate from field element + fr = Fr.random() + secret_key = SecretKey.from_fr(fr) + ``` + +=== "Rust" + ```rust + use blsttc::{SecretKey, Fr}; + use rand::thread_rng; + + // Generate from random seed + let secret_key = SecretKey::random(); + + // Generate from bytes + let bytes_data = b"some-32-byte-seed"; + let secret_key = SecretKey::from_bytes(bytes_data)?; + + // Generate from field element + let fr = Fr::random(); + let secret_key = SecretKey::from_fr(&fr); + ``` + +### Serialization + +=== "Python" + ```python + # Serialize keys and signatures + sk_bytes = secret_key.to_bytes() + pk_bytes = public_key.to_bytes() + sig_bytes = signature.to_bytes() + + # Deserialize + sk = SecretKey.from_bytes(sk_bytes) + pk = PublicKey.from_bytes(pk_bytes) + sig = Signature.from_bytes(sig_bytes) + ``` + +=== "Rust" + ```rust + // Serialize keys and signatures + let sk_bytes = secret_key.to_bytes(); + let pk_bytes = public_key.to_bytes(); + let sig_bytes = signature.to_bytes(); + + // Deserialize + let sk = SecretKey::from_bytes(&sk_bytes)?; + let pk = PublicKey::from_bytes(&pk_bytes)?; + let sig = Signature::from_bytes(&sig_bytes)?; + ``` + +## Error Handling + +=== "Python" + ```python + try: + # Operations that might fail + sk = SecretKey.from_bytes(invalid_bytes) + except ValueError as e: + print(f"Invalid key bytes: {e}") + + try: + # Signature verification + if not pk.verify(sig, msg): + print("Invalid signature") + except Exception as e: + print(f"Verification error: {e}") + ``` + +=== "Rust" + ```rust + use blsttc::error::Error; + + // Handle key generation errors + match SecretKey::from_bytes(invalid_bytes) { + Ok(sk) => println!("Key generated successfully"), + Err(Error::InvalidBytes) => println!("Invalid key bytes"), + Err(e) => println!("Other error: {}", e), + } + + // Handle signature verification + if !pk.verify(&sig, msg) { + println!("Invalid signature"); + } + ``` + +## Best Practices + +1. **Key Management** + - Securely store private keys + - Use strong random number generation + - Implement key rotation policies + +2. **Threshold Selection** + - Choose appropriate threshold values + - Consider fault tolerance requirements + - Balance security and availability + +3. **Performance** + - Cache public keys when possible + - Batch verify signatures when possible + - Use appropriate buffer sizes + +4. **Security** + - Validate all inputs + - Use secure random number generation + - Implement proper error handling + +## Common Use Cases + +1. **Distributed Key Generation** + - Generate keys for distributed systems + - Share keys among multiple parties + - Implement threshold cryptography + +2. **Signature Aggregation** + - Combine multiple signatures + - Reduce signature size + - Improve verification efficiency + +3. **Consensus Protocols** + - Implement Byzantine fault tolerance + - Create distributed voting systems + - Build secure multiparty computation diff --git a/docs/online-documentation/api/index.md b/docs/online-documentation/api/index.md new file mode 100644 index 0000000000..670862ecf9 --- /dev/null +++ b/docs/online-documentation/api/index.md @@ -0,0 +1,53 @@ +# API Reference Overview + +Autonomi provides several APIs for different aspects of the system: + +## Client API + +The [Autonomi Client API](autonomi-client/README.md) is the core library for interacting with the Autonomi network. It provides: + +- Data storage and retrieval +- Pointer management +- Graph operations +- File system operations +- Error handling + +## Node API + +The [Ant Node API](ant-node/README.md) allows you to run and manage nodes in the Autonomi network. Features include: + +- Node setup and configuration +- Network participation +- Storage management +- Reward collection +- Event handling + +## Cryptography APIs + +### BLS Threshold Crypto + +The [BLS Threshold Crypto API](blsttc/README.md) implements BLS (Boneh-Lynn-Shacham) threshold signatures, providing: + +- Secret key generation and sharing +- Signature creation and verification +- Threshold signature schemes +- Key aggregation + +### Self Encryption + +The [Self Encryption API](self-encryption/README.md) implements content-based encryption, offering: + +- Data-derived encryption +- Content deduplication +- Parallel processing +- Streaming interface + +## Language Support + +All APIs are available in multiple languages: + +- Python (3.8+) +- Rust (stable) +- Node.js (16+) + +Each API section includes language-specific installation instructions and code examples. diff --git a/docs/online-documentation/api/self-encryption/README.md b/docs/online-documentation/api/self-encryption/README.md new file mode 100644 index 0000000000..023e331a04 --- /dev/null +++ b/docs/online-documentation/api/self-encryption/README.md @@ -0,0 +1,267 @@ +# Self Encryption API Reference + +A file content self-encryptor that provides convergent encryption on file-based data. It produces a `DataMap` type and several chunks of encrypted data. Each chunk is up to 1MB in size and has an index and a name (SHA3-256 hash of the content), allowing chunks to be self-validating. + +## Installation + +=== "Python" + ```bash + pip install self-encryption + ``` + +=== "Rust" + ```toml + [dependencies] + self_encryption = "0.31.0" + ``` + +## Core Concepts + +### DataMap + +Holds the information required to recover the content of the encrypted file, stored as a vector of `ChunkInfo` (list of file's chunk hashes). Only files larger than 3072 bytes (3 * MIN_CHUNK_SIZE) can be self-encrypted. + +### Chunk Sizes + +- `MIN_CHUNK_SIZE`: 1 byte +- `MAX_CHUNK_SIZE`: 1 MiB (before compression) +- `MIN_ENCRYPTABLE_BYTES`: 3 bytes + +## Streaming Operations (Recommended) + +### Streaming File Encryption + +=== "Python" + ```python + from self_encryption import streaming_encrypt_from_file, ChunkStore + from pathlib import Path + from typing import Optional + + # Implement your chunk store + class MyChunkStore(ChunkStore): + def put(self, name: bytes, data: bytes) -> None: + # Store the chunk + pass + + def get(self, name: bytes) -> Optional[bytes]: + # Retrieve the chunk + pass + + # Create chunk store instance + store = MyChunkStore() + + # Encrypt file using streaming + file_path = Path("my_file.txt") + data_map = streaming_encrypt_from_file(file_path, store) + ``` + +=== "Rust" + ```rust + use self_encryption::{streaming_encrypt_from_file, ChunkStore}; + use std::path::Path; + + // Implement your chunk store + struct MyChunkStore { + // Your storage implementation + } + + impl ChunkStore for MyChunkStore { + fn put(&mut self, name: &[u8], data: &[u8]) -> Result<(), Error> { + // Store the chunk + } + + fn get(&self, name: &[u8]) -> Result, Error> { + // Retrieve the chunk + } + } + + // Create chunk store instance + let store = MyChunkStore::new(); + + // Encrypt file using streaming + let file_path = Path::new("my_file.txt"); + let data_map = streaming_encrypt_from_file(file_path, store).await?; + ``` + +### Streaming File Decryption + +=== "Python" + ```python + from self_encryption import streaming_decrypt_from_storage + from pathlib import Path + + # Decrypt to file using streaming + output_path = Path("decrypted_file.txt") + streaming_decrypt_from_storage(data_map, store, output_path) + ``` + +=== "Rust" + ```rust + use self_encryption::streaming_decrypt_from_storage; + use std::path::Path; + + // Decrypt to file using streaming + let output_path = Path::new("decrypted_file.txt"); + streaming_decrypt_from_storage(&data_map, store, output_path).await?; + ``` + +## In-Memory Operations (Small Files) + +### Basic Encryption/Decryption + +=== "Python" + ```python + from self_encryption import encrypt, decrypt + + # Encrypt bytes in memory + data = b"Small data to encrypt" + data_map, encrypted_chunks = encrypt(data) + + # Decrypt using retrieval function + def get_chunk(name: bytes) -> bytes: + # Retrieve chunk by name from your storage + return chunk_data + + decrypted = decrypt(data_map, get_chunk) + ``` + +=== "Rust" + ```rust + use self_encryption::{encrypt, decrypt}; + + // Encrypt bytes in memory + let data = b"Small data to encrypt"; + let (data_map, encrypted_chunks) = encrypt(data)?; + + // Decrypt using retrieval function + let decrypted = decrypt( + &data_map, + |name| { + // Retrieve chunk by name from your storage + Ok(chunk_data) + } + )?; + ``` + +## Chunk Store Implementations + +### In-Memory Store + +=== "Python" + ```python + from self_encryption import ChunkStore + from typing import Dict, Optional + + class MemoryStore(ChunkStore): + def __init__(self): + self.chunks: Dict[bytes, bytes] = {} + + def put(self, name: bytes, data: bytes) -> None: + self.chunks[name] = data + + def get(self, name: bytes) -> Optional[bytes]: + return self.chunks.get(name) + ``` + +=== "Rust" + ```rust + use std::collections::HashMap; + + struct MemoryStore { + chunks: HashMap, Vec>, + } + + impl ChunkStore for MemoryStore { + fn put(&mut self, name: &[u8], data: &[u8]) -> Result<(), Error> { + self.chunks.insert(name.to_vec(), data.to_vec()); + Ok(()) + } + + fn get(&self, name: &[u8]) -> Result, Error> { + self.chunks.get(name) + .cloned() + .ok_or(Error::NoSuchChunk) + } + } + ``` + +### Disk-Based Store + +=== "Python" + ```python + from pathlib import Path + from typing import Optional + import os + + class DiskStore(ChunkStore): + def __init__(self, root_dir: Path): + self.root_dir = root_dir + self.root_dir.mkdir(parents=True, exist_ok=True) + + def put(self, name: bytes, data: bytes) -> None: + path = self.root_dir / name.hex() + path.write_bytes(data) + + def get(self, name: bytes) -> Optional[bytes]: + path = self.root_dir / name.hex() + try: + return path.read_bytes() + except FileNotFoundError: + return None + ``` + +=== "Rust" + ```rust + use std::path::PathBuf; + use std::fs; + + struct DiskStore { + root_dir: PathBuf, + } + + impl ChunkStore for DiskStore { + fn put(&mut self, name: &[u8], data: &[u8]) -> Result<(), Error> { + let path = self.root_dir.join(hex::encode(name)); + fs::write(path, data)?; + Ok(()) + } + + fn get(&self, name: &[u8]) -> Result, Error> { + let path = self.root_dir.join(hex::encode(name)); + fs::read(path).map_err(|_| Error::NoSuchChunk) + } + } + + impl DiskStore { + fn new>(root: P) -> Self { + let root_dir = root.into(); + fs::create_dir_all(&root_dir).expect("Failed to create store directory"); + Self { root_dir } + } + } + ``` + +## Error Handling + +The library provides an `Error` enum for handling various error cases: + +```rust +pub enum Error { + NoSuchChunk, + ChunkTooSmall, + ChunkTooLarge, + InvalidChunkSize, + Io(std::io::Error), + Serialisation(Box), + Compression(std::io::Error), + // ... other variants +} +``` + +## Best Practices + +1. Use streaming operations (`streaming_encrypt_from_file` and `streaming_decrypt_from_storage`) for large files +2. Use basic `encrypt`/`decrypt` functions for small in-memory data +3. Implement proper error handling for chunk store operations +4. Verify chunks using their content hash when retrieving +5. Use parallel operations when available for better performance diff --git a/docs/online-documentation/getting-started/installation.md b/docs/online-documentation/getting-started/installation.md new file mode 100644 index 0000000000..b17130137a --- /dev/null +++ b/docs/online-documentation/getting-started/installation.md @@ -0,0 +1,110 @@ +# Installation Guide + +## Prerequisites + +- Rust (latest stable) +- Python 3.8 or higher +- Node.js 16 or higher + +## API-specific Installation + +Choose the APIs you need for your project: + +### Autonomi Client + +=== "Node.js" + ```bash + # Note: Package not yet published to npm + # Clone the repository and build from source + git clone https://github.com/dirvine/autonomi.git + cd autonomi + npm install + ``` + +=== "Python" + ```bash + pip install autonomi + ``` + +=== "Rust" + ```toml + # Add to Cargo.toml: + [dependencies] + autonomi = "0.3.1" + ``` + +### Ant Node + +=== "Python" + ```bash + pip install antnode + ``` + +=== "Rust" + ```toml + [dependencies] + ant-node = "0.3.2" + ``` + +### BLS Threshold Crypto + +=== "Python" + ```bash + pip install blsttc + ``` + +=== "Rust" + ```toml + [dependencies] + blsttc = "8.0.2" + ``` + +### Self Encryption + +=== "Python" + ```bash + pip install self-encryption + ``` + +=== "Rust" + ```toml + [dependencies] + self_encryption = "0.28.0" + ``` + +## Verifying Installation + +Test your installation by running a simple client initialization: + +=== "Node.js" + ```typescript + import { Client } from 'autonomi'; + + const client = await Client.initReadOnly(); + console.log('Client initialized successfully'); + ``` + +=== "Python" + ```python + from autonomi import Client + + client = Client.init_read_only() + print('Client initialized successfully') + ``` + +=== "Rust" + ```rust + use autonomi::Client; + + let client = Client::new_local().await?; + println!("Client initialized successfully"); + ``` + +## Next Steps + +- API References: + - [Autonomi Client](../api/autonomi-client/README.md) + - [Ant Node](../api/ant-node/README.md) + - [BLS Threshold Crypto](../api/blsttc/README.md) + - [Self Encryption](../api/self-encryption/README.md) +- [Local Network Setup](../guides/local_network.md) diff --git a/docs/online-documentation/guides/client_modes.md b/docs/online-documentation/guides/client_modes.md new file mode 100644 index 0000000000..4339cf2c62 --- /dev/null +++ b/docs/online-documentation/guides/client_modes.md @@ -0,0 +1,180 @@ +# Client Modes Guide + +This guide explains how to use Autonomi's client modes to browse the network (read-only) and optionally upgrade to write capabilities. + +## Overview + +Autonomi clients can operate in two modes: + +1. **Read-Only Mode**: Browse and read data from the network without requiring a wallet +2. **Read-Write Mode**: Full access to both read and write operations, requires a wallet + +## Read-Only Client + +A read-only client allows you to browse and read data from the network without needing a wallet or making payments. + +### Rust + +```rust +use autonomi::Client; + +// Initialize a read-only client +let client = Client::init_read_only().await?; + +// Verify it's read-only +assert!(!client.can_write()); +assert!(client.wallet().is_none()); + +// Read operations work normally +let data = client.get_bytes(address).await?; +let file = client.get_file(file_map, "output.txt").await?; +``` + +### TypeScript/JavaScript + +```typescript +import { Client } from '@autonomi/client'; + +// Initialize a read-only client +const client = await Client.connect({ + readOnly: true, + peers: ['/ip4/127.0.0.1/tcp/12000'] +}); + +// Read operations +const data = await client.dataGetPublic(address); +const list = await client.GraphEntryGet(listAddress); +``` + +### Python + +```python +from autonomi import Client + +# Initialize a read-only client +client = Client.new() + +# Read operations +data = client.get_bytes("safe://example_address") +file = client.get_file(file_map, "output.txt") +``` + +## Upgrading to Read-Write Mode + +You can upgrade a read-only client to read-write mode by adding a wallet. This enables write operations like storing data or updating graphs. + +### Rust + +```rust +use autonomi::{Client, EvmWallet}; + +// Start with a read-only client +let mut client = Client::init_read_only().await?; + +// Get a wallet (e.g., from a private key or create new) +let wallet = EvmWallet::from_private_key(private_key)?; + +// Upgrade to read-write mode +client.upgrade_to_read_write(wallet)?; + +// Now write operations are available +let address = client.store_bytes(data).await?; +``` + +### TypeScript/JavaScript + +```typescript +import { Client } from '@autonomi/client'; + +// Start with a read-only client +const client = await Client.connect({ + readOnly: true +}); + +// Upgrade with a wallet +await client.upgradeToReadWrite({ + type: 'wallet', + wallet: 'your_wallet_address' +}); + +// Now you can perform write operations +const address = await client.dataPutPublic( + Buffer.from('Hello World'), + { type: 'wallet', wallet: client.wallet } +); +``` + +### Python + +```python +from autonomi import Client, Wallet + +# Start with a read-only client +client = Client.new() + +# Create or import a wallet +wallet = Wallet.from_private_key("your_private_key") + +# Upgrade to read-write mode +client.upgrade_to_read_write(wallet) + +# Now write operations are available +address = client.store_bytes(b"Hello World") +``` + +## Write Operations + +The following operations require a wallet (read-write mode): + +- Storing public data (`dataPutPublic`) +- Creating/updating graphs (`GraphEntryPut`) +- Setting pointers (`pointerPut`) +- Writing to vaults (`writeBytesToVault`) +- Updating user data (`putUserDataToVault`) + +Attempting these operations in read-only mode will result in an error. + +## Best Practices + +1. **Start Read-Only**: Begin with a read-only client if you only need to read data. This is simpler and more secure since no wallet is needed. + +2. **Lazy Wallet Loading**: Only upgrade to read-write mode when you actually need to perform write operations. + +3. **Error Handling**: Always handle potential errors when upgrading modes or performing write operations: + +```typescript +try { + await client.upgradeToReadWrite(wallet); + await client.dataPutPublic(data, payment); +} catch (error) { + if (error.code === 'NO_WALLET') { + console.error('Write operation attempted without wallet'); + } else if (error.code === 'ALREADY_READ_WRITE') { + console.error('Client is already in read-write mode'); + } +} +``` + +4. **Check Capabilities**: Use the provided methods to check client capabilities: + +```rust +if client.can_write() { + // Perform write operation +} else { + // Handle read-only state +} +``` + +## Common Issues + +1. **Attempting Write Operations in Read-Only Mode** + - Error: `NO_WALLET` or `WriteAccessRequired` + - Solution: Upgrade to read-write mode by adding a wallet + +2. **Multiple Upgrade Attempts** + - Error: `ALREADY_READ_WRITE` + - Solution: Check client mode before attempting upgrade + +3. **Invalid Wallet** + - Error: `InvalidWallet` or `WalletError` + - Solution: Ensure wallet is properly initialized with valid credentials diff --git a/docs/online-documentation/guides/data_storage.md b/docs/online-documentation/guides/data_storage.md new file mode 100644 index 0000000000..4d8f90f744 --- /dev/null +++ b/docs/online-documentation/guides/data_storage.md @@ -0,0 +1,255 @@ +# Data Storage Guide + +This guide explains how Autonomi handles data storage, including self-encryption and scratchpad features. + +## Self-Encryption + +Self-encryption is a core feature that provides secure data storage by splitting and encrypting data into chunks. + +### How It Works + +1. Data is split into chunks +2. Each chunk is encrypted +3. A data map is created to track the chunks +4. Additional encryption layers are added for larger files + +### Usage Examples + +=== "Node.js" + ```typescript + import { Client } from '@autonomi/client'; + + async function storeEncryptedData(data: Uint8Array) { + const client = new Client(); + + // Data is automatically self-encrypted when stored + const address = await client.data_put_public(data); + console.log(`Data stored at: ${address}`); + + // Retrieve and decrypt data + const retrieved = await client.data_get_public(address); + console.log('Data retrieved successfully'); + } + ``` + +=== "Python" + ```python + from autonomi import Client + + async def store_encrypted_data(data: bytes): + client = Client() + + # Data is automatically self-encrypted when stored + address = await client.data_put_public(data) + print(f"Data stored at: {address}") + + # Retrieve and decrypt data + retrieved = await client.data_get_public(address) + print("Data retrieved successfully") + ``` + +=== "Rust" + ```rust + use autonomi::{Client, Bytes, Result}; + + async fn store_encrypted_data(data: Bytes) -> Result<()> { + let client = Client::new()?; + + // Data is automatically self-encrypted when stored + let address = client.data_put_public(data).await?; + println!("Data stored at: {}", address); + + // Retrieve and decrypt data + let retrieved = client.data_get_public(&address).await?; + println!("Data retrieved successfully"); + + Ok(()) + } + ``` + +## Scratchpad + +Scratchpad provides a mutable storage location for encrypted data with versioning support. + +### Features + +- Mutable data storage +- Version tracking with monotonic counter +- Owner-based access control +- Data encryption +- Signature verification + +### Usage Examples + +=== "Node.js" + ```typescript + import { Client, Scratchpad } from '@autonomi/client'; + + async function useScratchpad() { + const client = new Client(); + const secretKey = await client.generate_secret_key(); + + // Create or get existing scratchpad + const [scratchpad, isNew] = await client.get_or_create_scratchpad( + secretKey, + 42 // content type + ); + + // Update scratchpad data + const data = new TextEncoder().encode('Hello World'); + await client.update_scratchpad(scratchpad, data, secretKey); + + // Read scratchpad data + const retrieved = await client.get_scratchpad(scratchpad.address); + const decrypted = await client.decrypt_scratchpad(retrieved, secretKey); + console.log(new TextDecoder().decode(decrypted)); + } + ``` + +=== "Python" + ```python + from autonomi import Client, Scratchpad + + async def use_scratchpad(): + client = Client() + secret_key = client.generate_secret_key() + + # Create or get existing scratchpad + scratchpad, is_new = await client.get_or_create_scratchpad( + secret_key, + 42 # content type + ) + + # Update scratchpad data + data = b"Hello World" + await client.update_scratchpad(scratchpad, data, secret_key) + + # Read scratchpad data + retrieved = await client.get_scratchpad(scratchpad.address) + decrypted = await client.decrypt_scratchpad(retrieved, secret_key) + print(decrypted.decode()) + ``` + +=== "Rust" + ```rust + use autonomi::{Client, Scratchpad, SecretKey, Bytes, Result}; + + async fn use_scratchpad() -> Result<()> { + let client = Client::new()?; + let secret_key = SecretKey::random(); + + // Create or get existing scratchpad + let (mut scratchpad, is_new) = client + .get_or_create_scratchpad(&secret_key, 42) + .await?; + + // Update scratchpad data + let data = Bytes::from("Hello World"); + scratchpad.update_and_sign(data, &secret_key); + + // Store updated scratchpad + client.put_scratchpad(&scratchpad).await?; + + // Read scratchpad data + let retrieved = client.get_scratchpad(scratchpad.address()).await?; + let decrypted = retrieved.decrypt_data(&secret_key)?; + println!("Data: {}", String::from_utf8_lossy(&decrypted)); + + Ok(()) + } + ``` + +### Best Practices + +1. **Version Management** + - Always check the counter before updates + - Handle version conflicts appropriately + - Use monotonic counters for ordering + +2. **Security** + - Keep secret keys secure + - Verify signatures before trusting data + - Always encrypt sensitive data + +3. **Error Handling** + - Handle decryption failures gracefully + - Implement proper retry logic for network operations + - Validate data before storage + +4. **Performance** + - Cache frequently accessed data + - Batch updates when possible + - Monitor storage size + +## Implementation Details + +### Self-Encryption Process + +1. **Data Splitting** + + ```rust + // Internal process when storing data + let (data_map, chunks) = self_encryption::encrypt(data)?; + let (data_map_chunk, additional_chunks) = pack_data_map(data_map)?; + ``` + +2. **Chunk Management** + - Each chunk is stored separately + - Chunks are encrypted individually + - Data maps track chunk locations + +### Scratchpad Structure + +```rust +pub struct Scratchpad { + // Network address + address: ScratchpadAddress, + // Data type identifier + data_encoding: u64, + // Encrypted content + encrypted_data: Bytes, + // Version counter + counter: u64, + // Owner's signature + signature: Option, +} +``` + +## Advanced Topics + +### Custom Data Types + +You can use scratchpads to store any custom data type by implementing proper serialization: + +```rust +#[derive(Serialize, Deserialize)] +struct CustomData { + field1: String, + field2: u64, +} + +// Serialize before storing +let custom_data = CustomData { + field1: "test".into(), + field2: 42, +}; +let bytes = serde_json::to_vec(&custom_data)?; +scratchpad.update_and_sign(Bytes::from(bytes), &secret_key); +``` + +### Batch Operations + +For better performance when dealing with multiple data items: + +```rust +async fn batch_store(items: Vec) -> Result> { + let mut addresses = Vec::new(); + for item in items { + let (data_map_chunk, chunks) = encrypt(item)?; + // Store chunks in parallel + futures::future::join_all(chunks.iter().map(|c| store_chunk(c))).await; + addresses.push(data_map_chunk.address()); + } + Ok(addresses) +} +``` diff --git a/docs/online-documentation/guides/data_types.md b/docs/online-documentation/guides/data_types.md new file mode 100644 index 0000000000..d9cad3859e --- /dev/null +++ b/docs/online-documentation/guides/data_types.md @@ -0,0 +1,320 @@ +# Data Types Guide + +This guide explains the fundamental data types in Autonomi and how they can be used to build higher-level abstractions like files and directories. + +## Fundamental Data Types + +Autonomi provides four fundamental data types that serve as building blocks for all network operations. Each type is designed for specific use cases and together they provide a complete system for decentralized data management. + +### 1. Chunk + +Chunks are the foundation of secure data storage in Autonomi, primarily used as the output of self-encrypting files. This provides quantum-secure encryption for data at rest. + +```rust +// Store raw bytes as a chunk +let data = b"Hello, World!"; +let chunk_address = client.store_chunk(data).await?; + +// Retrieve chunk data +let retrieved = client.get_chunk(chunk_address).await?; +assert_eq!(data, retrieved); +``` + +Key characteristics: +- Quantum-secure encryption through self-encryption +- Immutable content +- Content-addressed (address is derived from data) +- Size-limited (maximum chunk size) +- Efficient for small to medium-sized data + +#### Self-Encryption Process +1. Data is split into fixed-size sections +2. Each section is encrypted using data from other sections +3. Results in multiple encrypted chunks +4. Original data can only be recovered with all chunks + +### 2. Pointer + +Pointers provide a fixed network address that can reference any other data type, including other pointers. They enable mutable data structures while maintaining stable addresses. + +```rust +// Create a pointer to some data +let pointer = client.create_pointer(target_address).await?; + +// Update pointer target +client.update_pointer(pointer.address(), new_target_address).await?; + +// Resolve pointer to get current target +let target = client.resolve_pointer(pointer.address()).await?; + +// Chain pointers for indirection +let pointer_to_pointer = client.create_pointer(pointer.address()).await?; +``` + +Key characteristics: +- Fixed network address +- Mutable reference capability +- Single owner (controlled by secret key) +- Version tracking with monotonic counter +- Atomic updates +- Support for pointer chains and indirection + +#### Common Use Cases +1. **Mutable Data References** + ```rust + // Update data while maintaining same address + let pointer = client.create_pointer(initial_data).await?; + client.update_pointer(pointer.address(), updated_data).await?; + ``` + +2. **Latest Version Publishing** + ```rust + // Point to latest version while maintaining history + let history = client.create_graph_entry().await?; + let latest = client.create_pointer(history.address()).await?; + ``` + +3. **Indirection and Redirection** + ```rust + // Create chain of pointers for flexible data management + let data_pointer = client.create_pointer(data).await?; + let redirect_pointer = client.create_pointer(data_pointer.address()).await?; + ``` + +### 3. GraphEntry + +Graphs in Autonomi are powerful structures of connected data on the network. They provide both historical tracking and CRDT-like properties. + +```rust +// Create a new graph entry, signed with a secret key +let graph_content = [42u8; 32]; // 32 bytes of content +let graph_entry = GraphEntry::new( + public_key, // Graph entry address and owner + vec![parent_pks], // Parent graph entries + graph_content, // 32 bytes graph content + vec![], // Optional outputs (links to other graph entries) + &secret_key // Secret key for signing +); + +// Calculate the cost to create a graph entry +let cost = client.graph_entry_cost(secret_key).await?; + +// Store the entry in the network +client.graph_entry_put(graph_entry, &wallet).await?; + +// Retrieve the entry from the network +let retrieved_entry = client.graph_entry_get(graph_entry.address()).await?; +``` + +Key characteristics: +- Decentralized Graph structure +- Each entry is signed by a unique key (sk) and addressed at that key (pk) +- CRDT-like conflict resolution +- Graph Traversal +- Can be used for value transfer (cryptocurrency-like) + +### 4. ScratchPad + +ScratchPad provides a flexible, unstructured data storage mechanism with CRDT properties through counter-based versioning. It's ideal for user account data, application configurations, and other frequently updated small data packets. + +```rust +// Create a scratchpad for user settings +let pad = client.create_scratchpad(ContentType::UserSettings).await?; + +// Update with encrypted data +let encrypted = encrypt_aes(settings_data, user_key)?; +client.update_scratchpad(pad.address(), encrypted).await?; + +// Read and decrypt current data +let encrypted = client.get_scratchpad(pad.address()).await?; +let settings = decrypt_aes(encrypted, user_key)?; +``` + +Key characteristics: +- Unstructured data storage +- Counter-based CRDT for conflict resolution +- Type-tagged content +- Support for user-managed encryption +- Efficient for frequent updates +- Ideal for small data packets + +#### Security Considerations + +1. **Encryption** + ```rust + // Example of AES encryption for scratchpad data + let key = generate_aes_key(); + let encrypted = aes_encrypt(data, key)?; + client.update_scratchpad(pad.address(), encrypted).await?; + ``` + +2. **Access Control** + ```rust + // Create encrypted scratchpad with access control + let (public_key, private_key) = generate_keypair(); + let encrypted_key = encrypt_with_public_key(aes_key, public_key); + let metadata = ScratchpadMetadata { + encrypted_key, + allowed_users: vec![public_key], + }; + client.create_scratchpad_with_access(metadata).await?; + ``` + +#### Common Applications + +1. **User Profiles** + ```rust + // Store encrypted user profile + let profile = UserProfile { name, settings }; + let encrypted = encrypt_profile(profile, user_key); + client.update_scratchpad(profile_pad, encrypted).await?; + ``` + +2. **Application State** + ```rust + // Maintain application configuration + let config = AppConfig { preferences, state }; + let pad = client.get_or_create_config_pad().await?; + client.update_scratchpad(pad, config).await?; + ``` + +3. **Temporary Storage** + ```rust + // Use as temporary workspace + let workspace = client.create_scratchpad(ContentType::Workspace).await?; + client.update_scratchpad(workspace, working_data).await?; + ``` + +## Higher-Level Abstractions + +These fundamental types can be combined to create higher-level data structures: + +### File System + +The Autonomi file system is built on top of these primitives: + +```rust +// Create a directory +let dir = client.create_directory("my_folder").await?; + +// Create a file +let file = client.create_file("example.txt", content).await?; + +// Add file to directory +client.add_to_directory(dir.address(), file.address()).await?; + +// List directory contents +let entries = client.list_directory(dir.address()).await?; +``` + +#### Files + +Files are implemented using a combination of chunks and pointers: + +- Large files are split into chunks +- File metadata stored in pointer +- Content addressing for deduplication + +```rust +// Store a large file +let file_map = client.store_file("large_file.dat").await?; + +// Read file contents +client.get_file(file_map, "output.dat").await?; +``` + +#### Directories + +Directories use graphs and pointers to maintain a mutable collection of entries: + +- GraphEntry stores directory entries +- Pointer maintains current directory state +- Hierarchical structure support + +```rust +// Create nested directory structure +let root = client.create_directory("/").await?; +let docs = client.create_directory("docs").await?; +client.add_to_directory(root.address(), docs.address()).await?; + +// List recursively +let tree = client.list_recursive(root.address()).await?; +``` + +## Common Patterns + +### Data Organization + +1. **Static Content** + - Use chunks for immutable data + - Content addressing enables deduplication + - Efficient for read-heavy workloads + +2. **Mutable References** + - Use pointers for updateable references + - Maintain stable addresses + - Version tracking built-in + +3. **Collections** + - Use graphs for linked data + - Efficient for append operations + - Good for logs and sequences + +4. **Temporary Storage** + - Use scratchpads for working data + - Frequent updates supported + - Type-tagged content + +### Best Practices + +1. **Choose the Right Type** + - Chunks for immutable data + - Pointers for mutable references + - GraphEntry for collections + - ScratchPads for temporary storage + +2. **Efficient Data Structures** + + ```rust + // Bad: Using chunks for frequently changing data + let chunk = client.store_chunk(changing_data).await?; + + // Good: Using scratchpad for frequently changing data + let pad = client.create_scratchpad(content_type).await?; + client.update_scratchpad(pad.address(), changing_data).await?; + ``` + +3. **Version Management** + + ```rust + // Track versions with pointers + let versions = Vec::new(); + versions.push(pointer.version()); + client.update_pointer(pointer.address(), new_data).await?; + versions.push(pointer.version()); + ``` + +4. **Error Handling** + + ```rust + match client.get_chunk(address).await { + Ok(data) => process_data(data), + Err(ChunkError::NotFound) => handle_missing_chunk(), + Err(ChunkError::InvalidSize) => handle_size_error(), + Err(e) => handle_other_error(e), + } + ``` + +## Common Issues + +1. **Size Limitations** + - Chunk size limits + - Solution: Split large data across multiple chunks + +2. **Update Conflicts** + - Concurrent pointer updates + - Solution: Use version checking + +3. **Performance** + - GraphEntry traversal costs + - Solution: Use appropriate data structures for access patterns diff --git a/docs/online-documentation/guides/evm_integration.md b/docs/online-documentation/guides/evm_integration.md new file mode 100644 index 0000000000..00564682ae --- /dev/null +++ b/docs/online-documentation/guides/evm_integration.md @@ -0,0 +1,49 @@ +# EVM Integration Guide + +This guide explains how to integrate Autonomi with EVM-compatible networks for testing and development. + +## Supported Networks + +- Local Hardhat network +- Sepolia testnet +- Goerli testnet +- Custom EVM networks + +## Setting Up Test Networks + +### Local Hardhat Network + +```bash +npx hardhat node +``` + +### Connecting to Test Networks + +```typescript +import { EvmNetwork } from '@autonomi/client'; + +const network = new EvmNetwork({ + chainId: 31337, // Local hardhat network + rpcUrl: 'http://127.0.0.1:8545' +}); +``` + +## Deploying Test Contracts + +1. Compile contracts +2. Deploy using Hardhat +3. Interact with contracts + +## Testing with Different Networks + +- Network configuration +- Gas settings +- Contract deployment +- Transaction handling + +## Best Practices + +- Error handling +- Gas optimization +- Security considerations +- Testing strategies diff --git a/docs/online-documentation/guides/local_development.md b/docs/online-documentation/guides/local_development.md new file mode 100644 index 0000000000..dacdb61022 --- /dev/null +++ b/docs/online-documentation/guides/local_development.md @@ -0,0 +1,132 @@ +# Local Development Environment + +This guide will help you set up a local development environment for building applications with Autonomi. We'll use a script that sets up a local network with all the necessary components for development and testing. + +## Prerequisites + +- Rust toolchain installed +- Git repository cloned +- Basic understanding of terminal/command line + +## Setup Script + +Save the following script as `start-local-network.sh` in your project root: + +```bash +#!/bin/bash +set -e + +# Configuration +NODE_DATA_DIR="$HOME/Library/Application Support/autonomi/node" +CLIENT_DATA_DIR="$HOME/Library/Application Support/autonomi/client" +EVM_PORT=4343 +EVM_RPC_URL="http://localhost:8545" +WALLET_ADDRESS="0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" +TOKEN_ADDRESS="0x5FbDB2315678afecb367f032d93F642f64180aa3" +LOG_LEVEL="info" +NODE_PORT=5000 + +# ... (rest of the script content) ... +``` + +Make the script executable: + +```bash +chmod +x start-local-network.sh +``` + +## Using the Development Environment + +1. Start the local network: + + ```bash + ./start-local-network.sh + ``` + +2. The script will: + - Build all necessary components (ant-node, evm-testnet, ant CLI) + - Start a local EVM testnet + - Start a local Autonomi node + - Set up the development environment + +3. Once running, you'll see information about: + - Network endpoints + - Environment variables + - Example commands + +## Environment Variables + +The following environment variables should be set for your development environment: + +```bash +export ANT_PEERS=/ip4/127.0.0.1/udp/5000/quic-v1 +export ANT_LOG=info +export CLIENT_DATA_PATH=$HOME/Library/Application Support/autonomi/client +``` + +## Example Usage + +### File Operations + +Upload a file: + +```bash +./target/debug/ant file upload path/to/file +``` + +Download a file: + +```bash +./target/debug/ant file download +``` + +### Node Operations + +Check node status: + +```bash +./target/debug/ant node status +``` + +Get wallet balance: + +```bash +./target/debug/ant wallet balance +``` + +## Development Tips + +1. **Local Testing**: The local network is perfect for testing your applications without affecting the main network. + +2. **Quick Iterations**: Changes to your application can be tested immediately without waiting for network confirmations. + +3. **Clean State**: Each time you start the network, it begins with a clean state, making it ideal for testing different scenarios. + +4. **Debugging**: The local environment provides detailed logs and quick feedback for debugging. + +## Customization + +You can customize the development environment by modifying the configuration variables at the top of the script: + +- `NODE_PORT`: Change the port the node listens on +- `LOG_LEVEL`: Adjust logging verbosity ("trace", "debug", "info", "warn", "error") +- `EVM_PORT`: Change the EVM testnet port +- Other settings as needed + +## Troubleshooting + +1. **Port Conflicts**: If you see port-in-use errors, modify the `NODE_PORT` or `EVM_PORT` in the script. + +2. **Process Cleanup**: If the script fails to start, ensure no old processes are running: + + ```bash + pkill -f "antnode" + pkill -f "evm-testnet" + ``` + +3. **Data Cleanup**: To start completely fresh, remove the data directories: + + ```bash + rm -rf "$HOME/Library/Application Support/autonomi/node" + rm -rf "$HOME/Library/Application Support/autonomi/client" + ``` diff --git a/docs/online-documentation/guides/local_network.md b/docs/online-documentation/guides/local_network.md new file mode 100644 index 0000000000..7ffa2291c2 --- /dev/null +++ b/docs/online-documentation/guides/local_network.md @@ -0,0 +1,302 @@ +# Local Network Setup Guide + +This guide explains how to set up and run a local Autonomi network for development and testing purposes. + +## Prerequisites + +- Rust toolchain (with `cargo` installed) +- Git (for cloning the repository) + +That's it! Everything else needed will be built from source. + +## Quick Start + +1. Clone the repository: + +```bash +git clone https://github.com/dirvine/autonomi +cd autonomi +``` + +2. Start the local network: + +```bash +./test-local.sh +``` + +This script will: + +- Build all necessary components +- Start a local EVM testnet +- Start a local Autonomi node +- Set up the development environment + +## Network Components + +The local network consists of: + +- An Autonomi node running in local mode +- A local EVM test network with pre-funded accounts +- Test wallets for development + +## Testing with EVM Networks + +The local EVM network provides a complete testing environment for blockchain interactions: + +### Pre-deployed Contracts + +The following contracts are automatically deployed: + +- Payment Vault Contract (`PaymentVaultNoProxy`) + - Handles data storage payments + - Manages token approvals and transfers + - Verifies payment proofs +- Test Token Contract (`TestToken`) + - ERC20 token for testing payments + - Pre-minted supply for test accounts + - Automatic approval for test wallets + +### Test Accounts + +Several accounts are pre-funded and ready to use: + +``` +Primary Test Account: +Address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 +Private Key: 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 +Balance: 10000 TEST tokens + +Secondary Test Account: +Address: 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 +Private Key: 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d +Balance: 1000 TEST tokens +``` + +### RPC Endpoint + +The local EVM network exposes an RPC endpoint at `http://localhost:8545` with: + +- Full JSON-RPC API support +- WebSocket subscriptions +- Low block time (1 second) +- Zero gas costs +- Instant transaction confirmations + +### Interacting with the Network + +#### JavaScript/TypeScript + +```typescript +import { ethers } from 'ethers'; + +// Connect to local network +const provider = new ethers.JsonRpcProvider('http://localhost:8545'); +const wallet = new ethers.Wallet(PRIVATE_KEY, provider); + +// Get contract instances +const paymentVault = new ethers.Contract( + PAYMENT_VAULT_ADDRESS, + PAYMENT_VAULT_ABI, + wallet +); + +// Interact with contracts +await paymentVault.getQuote([metrics]); +await paymentVault.payForQuotes(payments); +``` + +#### Python + +```python +from web3 import Web3 +from eth_account import Account + +# Connect to local network +w3 = Web3(Web3.HTTPProvider('http://localhost:8545')) +account = Account.from_key(PRIVATE_KEY) + +# Get contract instances +payment_vault = w3.eth.contract( + address=PAYMENT_VAULT_ADDRESS, + abi=PAYMENT_VAULT_ABI +) + +# Interact with contracts +payment_vault.functions.getQuote([metrics]).call() +payment_vault.functions.payForQuotes(payments).transact() +``` + +#### Rust + +```rust +use ethers::prelude::*; + +// Connect to local network +let provider = Provider::::try_from("http://localhost:8545")?; +let wallet = LocalWallet::from_bytes(&PRIVATE_KEY)?; +let client = SignerMiddleware::new(provider, wallet); + +// Get contract instances +let payment_vault = PaymentVault::new( + PAYMENT_VAULT_ADDRESS, + Arc::new(client) +); + +// Interact with contracts +payment_vault.get_quote(metrics).call().await?; +payment_vault.pay_for_quotes(payments).send().await?; +``` + +## Environment Variables + +The following environment variables are set up automatically: + +- `ANT_PEERS` - Local node endpoint +- `ANT_LOG` - Logging level +- `CLIENT_DATA_PATH` - Client data directory + +## Monitoring and Debugging + +### Logging + +#### Node Logs + +The Autonomi node generates detailed logs that can be controlled via `RUST_LOG`: + +```bash +# Trace level for maximum detail +RUST_LOG=trace ./test-local.sh + +# Focus on specific modules +RUST_LOG=autonomi=debug,ant_node=trace ./test-local.sh + +# Log locations: +- Node logs: $NODE_DATA_DIR/node.log +- EVM logs: $NODE_DATA_DIR/evm.log +``` + +#### Log Levels + +- `error`: Critical issues that need immediate attention +- `warn`: Important events that aren't failures +- `info`: General operational information +- `debug`: Detailed information for debugging +- `trace`: Very detailed protocol-level information + +#### Following Logs + +```bash +# Follow node logs +tail -f "$NODE_DATA_DIR/node.log" + +# Follow EVM logs +tail -f "$NODE_DATA_DIR/evm.log" + +# Filter for specific events +tail -f "$NODE_DATA_DIR/node.log" | grep "payment" +``` + +### Debugging + +#### Node Debugging + +Using `rust-lldb`: + +```bash +# Start node with debugger +rust-lldb target/debug/antnode -- --features test + +# Common commands: +b autonomi::client::payment::pay # Set breakpoint +r # Run +bt # Backtrace +p variable # Print variable +c # Continue +``` + +Using `rust-gdb`: + +```bash +# Start node with debugger +rust-gdb target/debug/antnode -- --features test + +# Common commands: +break autonomi::client::payment::pay # Set breakpoint +run # Run +backtrace # Show backtrace +print variable # Examine variable +continue # Continue execution +``` + +#### Network Monitoring + +Monitor network activity: + +```bash +# Watch network connections +netstat -an | grep 5000 # Default node port + +# Monitor network traffic +sudo tcpdump -i lo0 port 5000 + +# Check EVM network +curl -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://localhost:8545 +``` + +#### Contract Debugging + +Debug contract interactions: + +```bash +# Get payment vault state +cast call $PAYMENT_VAULT_ADDRESS \ + "payments(bytes32)" \ + $QUOTE_HASH \ + --rpc-url http://localhost:8545 + +# Watch for payment events +cast events $PAYMENT_VAULT_ADDRESS \ + --rpc-url http://localhost:8545 +``` + +## Common Issues and Solutions + +### Port Conflicts + +If you see port-in-use errors: + +1. Check if another instance is running +2. Use different ports in the script +3. Kill existing processes if needed + +### Build Issues + +1. Make sure Rust toolchain is up to date +2. Clean and rebuild: `cargo clean && cargo build` +3. Check for missing dependencies + +### Network Issues + +1. Verify the node is running +2. Check log output for errors +3. Ensure EVM testnet is accessible + +## Advanced Usage + +### Custom Configuration + +You can modify the test script to: + +- Change ports +- Adjust logging levels +- Configure node parameters + +### Multiple Nodes + +To run multiple nodes: + +1. Copy the script +2. Modify ports and directories +3. Run each instance separately diff --git a/docs/online-documentation/guides/payments.md b/docs/online-documentation/guides/payments.md new file mode 100644 index 0000000000..bf9b6fbb55 --- /dev/null +++ b/docs/online-documentation/guides/payments.md @@ -0,0 +1,277 @@ +# Payments Guide + +This guide explains how payments work in Autonomi, particularly for put operations that store data on the network. + +## Overview + +When storing data on the Autonomi network, you need to pay for the storage space. Payments are made using EVM-compatible tokens through a smart contract system. There are two ways to handle payments: + +1. Direct payment using an EVM wallet +2. Pre-paid operations using a receipt + +## Payment Options + +### Using an EVM Wallet + +The simplest way to pay for put operations is to use an EVM wallet: + +```python +# Python +from autonomi import Client, PaymentOption +from autonomi.evm import EvmWallet + +# Initialize client +client = Client() + +# Create or load a wallet +wallet = EvmWallet.create() # or load from private key +payment = PaymentOption.from_wallet(wallet) + +# Put data with wallet payment +data = b"Hello, World!" +address = client.data_put_public(data, payment) +``` + +```typescript +// Node.js +import { Client, PaymentOption } from '@autonomi/client'; +import { EvmWallet } from '@autonomi/evm'; + +// Initialize client +const client = new Client(); + +// Create or load a wallet +const wallet = EvmWallet.create(); // or load from private key +const payment = PaymentOption.fromWallet(wallet); + +// Put data with wallet payment +const data = Buffer.from("Hello, World!"); +const address = await client.dataPutPublic(data, payment); +``` + +```rust +// Rust +use autonomi::{Client, PaymentOption}; +use ant_evm::EvmWallet; + +// Initialize client +let client = Client::new()?; + +// Create or load a wallet +let wallet = EvmWallet::create()?; // or load from private key +let payment = wallet.into(); // Converts to PaymentOption + +// Put data with wallet payment +let data = b"Hello, World!".to_vec(); +let address = client.data_put_public(data.into(), payment).await?; +``` + +### Using Pre-paid Receipts + +For better efficiency when doing multiple put operations, you can pre-pay for storage and reuse the receipt: + +```python +# Python +from autonomi import Client, PaymentOption +from autonomi.evm import EvmWallet + +# Initialize client +client = Client() +wallet = EvmWallet.create() + +# Get receipt for multiple operations +data1 = b"First piece of data" +data2 = b"Second piece of data" + +# Create payment receipt +receipt = client.create_payment_receipt([data1, data2], wallet) +payment = PaymentOption.from_receipt(receipt) + +# Use receipt for puts +addr1 = client.data_put_public(data1, payment) +addr2 = client.data_put_public(data2, payment) +``` + +```typescript +// Node.js +import { Client, PaymentOption } from '@autonomi/client'; +import { EvmWallet } from '@autonomi/evm'; + +// Initialize client +const client = new Client(); +const wallet = EvmWallet.create(); + +// Get receipt for multiple operations +const data1 = Buffer.from("First piece of data"); +const data2 = Buffer.from("Second piece of data"); + +// Create payment receipt +const receipt = await client.createPaymentReceipt([data1, data2], wallet); +const payment = PaymentOption.fromReceipt(receipt); + +// Use receipt for puts +const addr1 = await client.dataPutPublic(data1, payment); +const addr2 = await client.dataPutPublic(data2, payment); +``` + +```rust +// Rust +use autonomi::{Client, PaymentOption}; +use ant_evm::EvmWallet; + +// Initialize client +let client = Client::new()?; +let wallet = EvmWallet::create()?; + +// Get receipt for multiple operations +let data1 = b"First piece of data".to_vec(); +let data2 = b"Second piece of data".to_vec(); + +// Create payment receipt +let receipt = client.create_payment_receipt( + vec![data1.clone(), data2.clone()].into_iter(), + &wallet +).await?; +let payment = receipt.into(); // Converts to PaymentOption + +// Use receipt for puts +let addr1 = client.data_put_public(data1.into(), payment.clone()).await?; +let addr2 = client.data_put_public(data2.into(), payment).await?; +``` + +## Cost Calculation + +The cost of storing data depends on several factors: + +- Size of the data +- Network density +- Storage duration +- Current network conditions + +You can calculate the cost before performing a put operation: + +```python +# Python +cost = client.calculate_storage_cost(data) +print(f"Storage will cost {cost} tokens") +``` + +```typescript +// Node.js +const cost = await client.calculateStorageCost(data); +console.log(`Storage will cost ${cost} tokens`); +``` + +```rust +// Rust +let cost = client.calculate_storage_cost(&data).await?; +println!("Storage will cost {} tokens", cost); +``` + +## Token Management + +Before you can pay for storage, you need to ensure your wallet has sufficient tokens and has approved the payment contract to spend them: + +```python +# Python +# Check balance +balance = wallet.get_balance() + +# Approve tokens if needed +if not wallet.has_approved_tokens(): + wallet.approve_tokens() +``` + +```typescript +// Node.js +// Check balance +const balance = await wallet.getBalance(); + +// Approve tokens if needed +if (!await wallet.hasApprovedTokens()) { + await wallet.approveTokens(); +} +``` + +```rust +// Rust +// Check balance +let balance = wallet.get_balance().await?; + +// Approve tokens if needed +if !wallet.has_approved_tokens().await? { + wallet.approve_tokens().await?; +} +``` + +## Error Handling + +Common payment-related errors you might encounter: + +1. `InsufficientBalance` - Wallet doesn't have enough tokens +2. `TokenNotApproved` - Token spending not approved for the payment contract +3. `PaymentExpired` - Payment quote has expired (when using receipts) +4. `PaymentVerificationFailed` - Payment verification failed on the network + +Example error handling: + +```python +# Python +try: + address = client.data_put_public(data, payment) +except InsufficientBalance: + print("Not enough tokens in wallet") +except TokenNotApproved: + print("Need to approve token spending") +except PaymentError as e: + print(f"Payment failed: {e}") +``` + +```typescript +// Node.js +try { + const address = await client.dataPutPublic(data, payment); +} catch (e) { + if (e instanceof InsufficientBalance) { + console.log("Not enough tokens in wallet"); + } else if (e instanceof TokenNotApproved) { + console.log("Need to approve token spending"); + } else { + console.log(`Payment failed: ${e}`); + } +} +``` + +```rust +// Rust +match client.data_put_public(data.into(), payment).await { + Err(PutError::InsufficientBalance) => { + println!("Not enough tokens in wallet"); + } + Err(PutError::TokenNotApproved) => { + println!("Need to approve token spending"); + } + Err(e) => { + println!("Payment failed: {}", e); + } + Ok(address) => { + println!("Data stored at {}", address); + } +} +``` + +## Best Practices + +1. **Pre-approve Tokens**: Approve token spending before starting put operations to avoid extra transactions. + +2. **Use Receipts**: When doing multiple put operations, use receipts to avoid making separate payments for each operation. + +3. **Check Costs**: Always check storage costs before proceeding with large data uploads. + +4. **Handle Errors**: Implement proper error handling for payment-related issues. + +5. **Monitor Balance**: Keep track of your wallet balance to ensure sufficient funds for operations. + +## Testing Payments + +When testing your application, you can use the local development environment which provides a test EVM network with pre-funded wallets. See the [Local Development Guide](local_development.md) for details. diff --git a/docs/online-documentation/guides/testing_guide.md b/docs/online-documentation/guides/testing_guide.md new file mode 100644 index 0000000000..7a78d86e58 --- /dev/null +++ b/docs/online-documentation/guides/testing_guide.md @@ -0,0 +1,111 @@ +# Testing Guide + +This guide covers testing strategies for Autonomi applications across different languages and environments. + +## Test Environment Setup + +### Node.js + +```bash +npm install --save-dev jest @types/jest ts-jest +``` + +### Python + +```bash +pip install pytest pytest-asyncio +``` + +### Rust + +```bash +cargo install cargo-test +``` + +## Writing Tests + +### Node.js Example + +```typescript +import { Client, GraphEntry } from '@autonomi/client'; + +describe('GraphEntry Operations', () => { + let client: Client; + + beforeEach(() => { + client = new Client(); + }); + + test('should store and retrieve graph', async () => { + const list = new GraphEntry(); + list.append("test data"); + + const address = await client.GraphEntryPut(list); + const retrieved = await client.GraphEntryGet(address); + + expect(retrieved.toString()).toBe("test data"); + }); +}); +``` + +### Python Example + +```python +import pytest +from autonomi import Client, GraphEntry + +@pytest.mark.asyncio +async def test_graph_entry_operations(): + client = Client() + + # Create and store list + entry_obj = GraphEntry() + entry_obj.append("test data") + + address = await client.graph_entry_put(entry_obj) + retrieved = await client.graph_entry_get(address) + + assert str(retrieved) == "test data" +``` + +### Rust Example + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_graph_entry_operations() { + let client = Client::new(); + + let mut entry = GraphEntry::new(); + entry.append("test data"); + + let address = client.graph_entry_put(&entry).unwrap(); + let retrieved = client.graph_entry_get(&address).unwrap(); + + assert_eq!(retrieved.to_string(), "test data"); + } +} +``` + +## Test Categories + +1. Unit Tests +2. Integration Tests +3. Network Tests +4. EVM Integration Tests + +## CI/CD Integration + +- GitHub Actions configuration +- Test automation +- Coverage reporting + +## Best Practices + +- Test isolation +- Mock network calls +- Error scenarios +- Performance testing diff --git a/docs/online-documentation/index.md b/docs/online-documentation/index.md new file mode 100644 index 0000000000..05062bdd30 --- /dev/null +++ b/docs/online-documentation/index.md @@ -0,0 +1,133 @@ +# Autonomi Documentation + +Welcome to the Autonomi documentation! This guide will help you get started with using the Autonomi network client. + +## What is Autonomi? + +Autonomi is a decentralised data and communications platform designed to provide complete privacy, security, and freedom by distributing data across a peer-to-peer network, rather than relying on centralised servers. Through end-to-end encryption, self-authentication, and the allocation of storage and bandwidth from users’ own devices, it seeks to create an autonomous, self-sustaining system where data ownership remains firmly in the hands of individuals rather than corporations. + +## Quick Links + +- [Installation Guide](getting-started/installation.md) +- Core Concepts: + - [Data Types](guides/data_types.md) - Understanding the fundamental data structures + - [Client Modes](guides/client_modes.md) - Different operational modes of the client + - [Data Storage](guides/data_storage.md) - How data is stored and retrieved + - [Local Network Setup](guides/local_network.md) - Setting up a local development environment + +### API References + +- [Autonomi Client](api/autonomi-client/README.md) - Core client library for network operations +- [Ant Node](api/ant-node/README.md) - Node implementation for network participation +- [BLS Threshold Crypto](api/blsttc/README.md) - Threshold cryptography implementation +- [Self Encryption](api/self-encryption/README.md) - Content-based encryption library +- Developer Resources: + + +## Language Support + +Autonomi provides client libraries for multiple languages: + +=== "Node.js" + ```typescript + import { Client } from 'autonomi'; + + const client = new Client(); + await client.connect(); + ``` + +=== "Python" + ```python + from autonomi import Client + + client = Client() + await client.connect() + ``` + +=== "Rust" + ```rust + use autonomi::Client; + + let client = Client::new()?; + ``` + +## Building from Source + +=== "Python (using Maturin & uv)" + ```bash + # Install build dependencies + curl -LsSf | sh + uv pip install maturin + + # Clone the repository + git clone https://github.com/dirvine/autonomi.git + cd autonomi + + # Create and activate virtual environment + uv venv + source .venv/bin/activate # Unix + # or + .venv\Scripts\activate # Windows + + # Build and install the package + cd python + maturin develop + + # Install dependencies + uv pip install -r requirements.txt + ``` + +=== "Node.js" + ```bash + # Install build dependencies + npm install -g node-gyp + + # Clone the repository + git clone https://github.com/dirvine/autonomi.git + cd autonomi + + # Build the Node.js bindings + cd nodejs + npm install + npm run build + + # Link for local development + npm link + ``` + +=== "Rust" + ```bash + # Clone the repository + git clone + cd autonomi + + # Build the project + cargo build --release + + # Run tests + cargo test --all-features + + # Install locally + cargo install --path . + ``` + +## Contributing + +We welcome contributions! Here's how you can help: + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +For more details, see our [Contributing Guide](https://github.com/dirvine/autonomi/blob/main/CONTRIBUTING.md). + +## Getting Help + +- [GitHub Issues](https://github.com/dirvine/autonomi/issues) +- API References: + - [Autonomi Client](api/autonomi-client/README.md) + - [Ant Node](api/ant-node/README.md) + - [BLS Threshold Crypto](api/blsttc/README.md) + - [Self Encryption](api/self-encryption/README.md) +- [Testing Guide](guides/testing_guide.md) diff --git a/docs/pointer_design_doc.md b/docs/pointer_design_doc.md new file mode 100644 index 0000000000..42034ab41c --- /dev/null +++ b/docs/pointer_design_doc.md @@ -0,0 +1,75 @@ +# Pointer Data Type Design Document + +## Overview + +The `Pointer` data type is designed to represent a reference to a `GraphEntry` in the system. It will include metadata such as the owner, a counter, and a signature to ensure data integrity and authenticity. + +## Structure + +```rust +struct Pointer { + owner: PubKey, // This is the address of this data type + counter: U32, + target: PointerTarget, // Can be PointerAddress, GraphEntryAddress, ChunksAddress, or ScratchpadAddress + signature: Sig, // Signature of counter and pointer (and target) +} +``` + +## Pointer Target + +The `PointerTarget` enum will define the possible target types for a `Pointer`: + +```rust +enum PointerTarget { + PointerAddress(PointerAddress), + GraphEntryAddress(GraphEntryAddress), + ChunkAddress(ChunkAddress), + ScratchpadAddress(ScratchpadAddress), +} +``` + +## Detailed Implementation and Testing Strategy + +1. **Define the `Pointer` Struct**: + - Implement the `Pointer` struct in a new Rust file alongside `graph_entry.rs`. + - **Testing**: Write unit tests to ensure the struct is correctly defined and can be instantiated. + +2. **Address Handling**: + - Implement address handling similar to `GraphEntryAddress`. + - **Testing**: Verify address conversion and serialization through unit tests. + +3. **Integration with `record_store.rs`**: + - Ensure that the `Pointer` type is properly integrated into the `record_store.rs` to handle storage and retrieval operations. + - **Testing**: Use integration tests to confirm that `Pointer` records can be stored and retrieved correctly. + +4. **Signature Verification**: + - Implement methods to sign and verify the `Pointer` data using the owner's private key. + - **Testing**: Write tests to validate the signature creation and verification process. + +5. **Output Handling**: + - The `Pointer` will point to a `GraphEntry`, and the `GraphEntry` output will be used as the value. If there is more than one output, the return will be a vector of possible values. + - **Testing**: Test the output handling logic to ensure it returns the correct values. + +6. **Integration with ant-networking**: + - Implement methods to serialize and deserialize `Pointer` records, similar to how `GraphEntry` records are handled. + - Ensure that the `Pointer` type is supported in the `NodeRecordStore` for storage and retrieval operations. + - **Testing**: Conduct end-to-end tests to verify the integration with `ant-networking`. + +7. **Payment Handling**: + - Introduce `RecordKind::PointerWithPayment` to handle `Pointer` records with payments. + - Implement logic to process `Pointer` records with payments, similar to `GraphEntryWithPayment`. + - **Testing**: Test the payment processing logic to ensure it handles payments correctly. + +8. **Documentation and Review**: + - Update documentation to reflect the new `Pointer` type and its usage. + - Conduct code reviews to ensure quality and adherence to best practices. + +## Next Steps + +- Develop a detailed implementation plan for each component. +- Identify any additional dependencies or libraries required. +- Plan for testing and validation of the `Pointer` data type. + +## Conclusion + +The `Pointer` data type will enhance the system's ability to reference and manage `GraphEntry` structures efficiently. Further details will be added as the implementation progresses. diff --git a/evm-testnet/Cargo.toml b/evm-testnet/Cargo.toml index fcff5a809b..8dac620dbf 100644 --- a/evm-testnet/Cargo.toml +++ b/evm-testnet/Cargo.toml @@ -6,13 +6,13 @@ homepage = "https://maidsafe.net" license = "GPL-3.0" name = "evm-testnet" repository = "https://github.com/maidsafe/autonomi" -version = "0.1.8" +version = "0.1.9" [dependencies] -ant-evm = { path = "../ant-evm", version = "0.1.8" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } clap = { version = "4.5", features = ["derive"] } dirs-next = "~2.0.0" -evmlib = { path = "../evmlib", version = "0.1.8" } +evmlib = { path = "../evmlib", version = "0.1.9" } tokio = { version = "1.40", features = ["rt-multi-thread", "signal"] } [lints] diff --git a/evm-testnet/src/main.rs b/evm-testnet/src/main.rs index f865cb8983..e07864700a 100644 --- a/evm-testnet/src/main.rs +++ b/evm-testnet/src/main.rs @@ -164,7 +164,7 @@ impl TestnetData { ); std::fs::write(&csv_path, csv).expect("Could not write to evm_testnet_data.csv file"); println!("EVM testnet data saved to: {csv_path:?}"); - println!("When running the Node or CLI with --feature=local, it will automatically use this network by loading the EVM Network's info from the CSV file."); + println!("When running the Node or CLI in local mode, it will automatically use this network by loading the EVM Network's info from the CSV file."); println!(); } diff --git a/evmlib/Cargo.toml b/evmlib/Cargo.toml index 486516de27..61d71e3493 100644 --- a/evmlib/Cargo.toml +++ b/evmlib/Cargo.toml @@ -5,26 +5,24 @@ edition = "2021" homepage = "https://maidsafe.net" license = "GPL-3.0" name = "evmlib" -repository = "https://github.com/maidsafe/safe_network" -version = "0.1.8" +repository = "https://github.com/maidsafe/autonomi" +version = "0.1.9" [features] -wasm-bindgen = ["alloy/wasm-bindgen"] -local = [] external-signer = [] [dependencies] alloy = { version = "0.7.3", default-features = false, features = ["contract", "json-rpc", "network", "node-bindings", "provider-http", "reqwest-rustls-tls", "rpc-client", "rpc-types", "signer-local", "std"] } dirs-next = "~2.0.0" -serde = "=1.0.210" +serde = "1" serde_with = { version = "3.11.0", features = ["macros"] } thiserror = "1.0" tracing = { version = "~0.1.26" } tokio = "1.38.0" rand = "0.8.5" -[target.'cfg(target_arch = "wasm32")'.dependencies] -getrandom = { version = "0.2.12", features = ["js"] } +[dev-dependencies] +tracing-subscriber = { version = "0.3", features = ["env-filter"] } [lints] workspace = true diff --git a/evmlib/abi/IPaymentVault.json b/evmlib/abi/IPaymentVaultV2.json similarity index 74% rename from evmlib/abi/IPaymentVault.json rename to evmlib/abi/IPaymentVaultV2.json index d1ca0a9f67..d6a3105b09 100644 --- a/evmlib/abi/IPaymentVault.json +++ b/evmlib/abi/IPaymentVaultV2.json @@ -43,11 +43,38 @@ "inputs": [ { "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "dataSize", + "type": "uint256" + }, { "internalType": "uint256", "name": "closeRecordsStored", "type": "uint256" }, + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "records", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.Record[]", + "name": "recordsPerType", + "type": "tuple[]" + }, { "internalType": "uint256", "name": "maxRecords", @@ -126,11 +153,38 @@ "components": [ { "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "dataSize", + "type": "uint256" + }, { "internalType": "uint256", "name": "closeRecordsStored", "type": "uint256" }, + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "records", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.Record[]", + "name": "recordsPerType", + "type": "tuple[]" + }, { "internalType": "uint256", "name": "maxRecords", diff --git a/evmlib/artifacts/PaymentVaultNoProxy.json b/evmlib/artifacts/PaymentVaultNoProxy.json deleted file mode 100644 index 9b006d274e..0000000000 --- a/evmlib/artifacts/PaymentVaultNoProxy.json +++ /dev/null @@ -1,339 +0,0 @@ -{ - "_format": "hh-sol-artifact-1", - "contractName": "PaymentVault", - "sourceName": "contracts/PaymentVaultNoProxy.sol", - "abi": [ - { - "inputs": [ - { - "internalType": "contract IERC20", - "name": "_antToken", - "type": "address" - }, - { - "internalType": "uint256", - "name": "_batchLimit", - "type": "uint256" - } - ], - "stateMutability": "nonpayable", - "type": "constructor" - }, - { - "inputs": [ - { - "internalType": "address", - "name": "target", - "type": "address" - } - ], - "name": "AddressEmptyCode", - "type": "error" - }, - { - "inputs": [ - { - "internalType": "address", - "name": "account", - "type": "address" - } - ], - "name": "AddressInsufficientBalance", - "type": "error" - }, - { - "inputs": [], - "name": "AntTokenNull", - "type": "error" - }, - { - "inputs": [], - "name": "BatchLimitExceeded", - "type": "error" - }, - { - "inputs": [], - "name": "FailedInnerCall", - "type": "error" - }, - { - "inputs": [], - "name": "InvalidInputLength", - "type": "error" - }, - { - "inputs": [ - { - "internalType": "address", - "name": "token", - "type": "address" - } - ], - "name": "SafeERC20FailedOperation", - "type": "error" - }, - { - "anonymous": false, - "inputs": [ - { - "indexed": true, - "internalType": "address", - "name": "rewardsAddress", - "type": "address" - }, - { - "indexed": true, - "internalType": "uint256", - "name": "amount", - "type": "uint256" - }, - { - "indexed": true, - "internalType": "bytes32", - "name": "quoteHash", - "type": "bytes32" - } - ], - "name": "DataPaymentMade", - "type": "event" - }, - { - "inputs": [], - "name": "antToken", - "outputs": [ - { - "internalType": "contract IERC20", - "name": "", - "type": "address" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "batchLimit", - "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { - "internalType": "uint256", - "name": "closeRecordsStored", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "maxRecords", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "receivedPaymentCount", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "liveTime", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "networkDensity", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "networkSize", - "type": "uint256" - } - ], - "internalType": "struct IPaymentVault.QuotingMetrics[]", - "name": "", - "type": "tuple[]" - } - ], - "name": "getQuote", - "outputs": [ - { - "internalType": "uint256[]", - "name": "prices", - "type": "uint256[]" - } - ], - "stateMutability": "pure", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { - "internalType": "address", - "name": "rewardsAddress", - "type": "address" - }, - { - "internalType": "uint256", - "name": "amount", - "type": "uint256" - }, - { - "internalType": "bytes32", - "name": "quoteHash", - "type": "bytes32" - } - ], - "internalType": "struct IPaymentVault.DataPayment[]", - "name": "_payments", - "type": "tuple[]" - } - ], - "name": "payForQuotes", - "outputs": [], - "stateMutability": "nonpayable", - "type": "function" - }, - { - "inputs": [ - { - "internalType": "bytes32", - "name": "", - "type": "bytes32" - } - ], - "name": "payments", - "outputs": [ - { - "internalType": "address", - "name": "rewardsAddress", - "type": "address" - }, - { - "internalType": "uint256", - "name": "amount", - "type": "uint256" - }, - { - "internalType": "bytes32", - "name": "quoteHash", - "type": "bytes32" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "requiredPaymentVerificationLength", - "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { - "components": [ - { - "internalType": "uint256", - "name": "closeRecordsStored", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "maxRecords", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "receivedPaymentCount", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "liveTime", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "networkDensity", - "type": "uint256" - }, - { - "internalType": "uint256", - "name": "networkSize", - "type": "uint256" - } - ], - "internalType": "struct IPaymentVault.QuotingMetrics", - "name": "metrics", - "type": "tuple" - }, - { - "internalType": "address", - "name": "rewardsAddress", - "type": "address" - }, - { - "internalType": "bytes32", - "name": "quoteHash", - "type": "bytes32" - } - ], - "internalType": "struct IPaymentVault.PaymentVerification[]", - "name": "_payments", - "type": "tuple[]" - } - ], - "name": "verifyPayment", - "outputs": [ - { - "components": [ - { - "internalType": "bytes32", - "name": "quoteHash", - "type": "bytes32" - }, - { - "internalType": "uint256", - "name": "amountPaid", - "type": "uint256" - }, - { - "internalType": "bool", - "name": "isValid", - "type": "bool" - } - ], - "internalType": "struct IPaymentVault.PaymentVerificationResult[3]", - "name": "verificationResults", - "type": "tuple[3]" - } - ], - "stateMutability": "view", - "type": "function" - } - ], - "bytecode": "0x6080604052348015600f57600080fd5b50604051610dce380380610dce833981016040819052602c91607f565b6001600160a01b038216605257604051632d06160b60e21b815260040160405180910390fd5b600180546001600160a01b0319166001600160a01b039390931692909217909155600055600560035560b7565b60008060408385031215609157600080fd5b82516001600160a01b038116811460a757600080fd5b6020939093015192949293505050565b610d08806100c66000396000f3fe608060405234801561001057600080fd5b506004361061007d5760003560e01c806380a38d971161005b57806380a38d9714610128578063b6c2141b14610148578063c7170bb61461015d578063f69c32cd1461016657600080fd5b80630716326d14610082578063474740b1146100e65780634ec42e8e146100fd575b600080fd5b6100bc6100903660046108fc565b60026020819052600091825260409091208054600182015491909201546001600160a01b039092169183565b604080516001600160a01b0390941684526020840192909252908201526060015b60405180910390f35b6100ef60005481565b6040519081526020016100dd565b600154610110906001600160a01b031681565b6040516001600160a01b0390911681526020016100dd565b61013b610136366004610915565b610186565b6040516100dd919061098c565b61015b6101563660046109cf565b6101d3565b005b6100ef60035481565b610179610174366004610a36565b6102c3565b6040516100dd9190610a9d565b60408051600180825281830190925260609160009190602080830190803683370190505090506001816000815181106101c1576101c1610aed565b60209081029190910101529392505050565b60005481908111156101f857604051630d67f41160e21b815260040160405180910390fd5b60005b818110156102bd573684848381811061021657610216610aed565b60600291909101915061024a9050336102326020840184610b28565b6001546001600160a01b03169190602085013561045c565b604080820135600090815260026020522081906102678282610b45565b505060408101356020820180359061027f9084610b28565b6001600160a01b03167ff998960b1c6f0e0e89b7bbe6b6fbf3e03e6f08eee5b8430877d8adb8e149d58060405160405180910390a4506001016101fb565b50505050565b6102cb610838565b60035482146102ed57604051637db491eb60e01b815260040160405180910390fd5b60006102f984846104b6565b905060005b60038110156104545760006002600084846003811061031f5761031f610aed565b602090810291909101516040908101518352828201939093529082016000908120835160608101855281546001600160a01b0316815260018201549381018490526002909101549381018490529350911515919015159085856003811061038857610388610aed565b6020020151602001516001600160a01b031684600001516001600160a01b03161480156103d9575060008686600381106103c4576103c4610aed565b6020020151602001516001600160a01b031614155b9050600060405180606001604052808888600381106103fa576103fa610aed565b60200201516040015181526020018660200151815260200185801561041c5750845b80156104255750835b1515905290508088876003811061043e5761043e610aed565b60200201525050600190930192506102fe915050565b505092915050565b604080516001600160a01b0385811660248301528416604482015260648082018490528251808303909101815260849091019091526020810180516001600160e01b03166323b872dd60e01b1790526102bd908590610691565b6104be610877565b60005b8281101561068a576000600260008686858181106104e1576104e1610aed565b9050610100020160e0013581526020019081526020016000206040518060600160405290816000820160009054906101000a90046001600160a01b03166001600160a01b03166001600160a01b03168152602001600182015481526020016002820154815250509050600260008460006003811061056157610561610aed565b602002015160400151815260200190815260200160002060010154816020015111156105cd576020830180516040850152835190528484838181106105a8576105a8610aed565b905061010002018036038101906105bf9190610beb565b8360005b6020020152610681565b602080840151604090810151600090815260028352206001015490820151111561062c576020830151604084015284848381811061060d5761060d610aed565b905061010002018036038101906106249190610beb565b8360016105c3565b604080840151810151600090815260026020908152919020600101549082015111156106815784848381811061066457610664610aed565b9050610100020180360381019061067b9190610beb565b60408401525b506001016104c1565b5092915050565b60006106a66001600160a01b038416836106fe565b905080516000141580156106cb5750808060200190518101906106c99190610c81565b155b156106f957604051635274afe760e01b81526001600160a01b03841660048201526024015b60405180910390fd5b505050565b606061070c83836000610713565b9392505050565b6060814710156107385760405163cd78605960e01b81523060048201526024016106f0565b600080856001600160a01b031684866040516107549190610ca3565b60006040518083038185875af1925050503d8060008114610791576040519150601f19603f3d011682016040523d82523d6000602084013e610796565b606091505b50915091506107a68683836107b0565b9695505050505050565b6060826107c5576107c08261080c565b61070c565b81511580156107dc57506001600160a01b0384163b155b1561080557604051639996b31560e01b81526001600160a01b03851660048201526024016106f0565b508061070c565b80511561081c5780518082602001fd5b604051630a12f52160e11b815260040160405180910390fd5b50565b60405180606001604052806003905b60408051606081018252600080825260208083018290529282015282526000199092019101816108475790505090565b60405180606001604052806003905b61088e6108a4565b8152602001906001900390816108865790505090565b60405180606001604052806108e86040518060c001604052806000815260200160008152602001600081526020016000815260200160008152602001600081525090565b815260006020820181905260409091015290565b60006020828403121561090e57600080fd5b5035919050565b6000806020838503121561092857600080fd5b823567ffffffffffffffff81111561093f57600080fd5b8301601f8101851361095057600080fd5b803567ffffffffffffffff81111561096757600080fd5b85602060c08302840101111561097c57600080fd5b6020919091019590945092505050565b602080825282518282018190526000918401906040840190835b818110156109c45783518352602093840193909201916001016109a6565b509095945050505050565b600080602083850312156109e257600080fd5b823567ffffffffffffffff8111156109f957600080fd5b8301601f81018513610a0a57600080fd5b803567ffffffffffffffff811115610a2157600080fd5b85602060608302840101111561097c57600080fd5b60008060208385031215610a4957600080fd5b823567ffffffffffffffff811115610a6057600080fd5b8301601f81018513610a7157600080fd5b803567ffffffffffffffff811115610a8857600080fd5b8560208260081b840101111561097c57600080fd5b6101208101818360005b6003811015610ae4578151805184526020810151602085015260408101511515604085015250606083019250602082019150600181019050610aa7565b50505092915050565b634e487b7160e01b600052603260045260246000fd5b6001600160a01b038116811461083557600080fd5b8035610b2381610b03565b919050565b600060208284031215610b3a57600080fd5b813561070c81610b03565b8135610b5081610b03565b81546001600160a01b0319166001600160a01b039190911617815560208201356001820155604090910135600290910155565b6040516060810167ffffffffffffffff81118282101715610bb457634e487b7160e01b600052604160045260246000fd5b60405290565b60405160c0810167ffffffffffffffff81118282101715610bb457634e487b7160e01b600052604160045260246000fd5b600081830361010081128015610c0057600080fd5b506000610c0b610b83565b60c0831215610c18578182fd5b610c20610bba565b853581526020808701359082015260408087013590820152606080870135908201526080808701359082015260a080870135908201528082529250610c6760c08601610b18565b602082015260e09490940135604085015250919392505050565b600060208284031215610c9357600080fd5b8151801515811461070c57600080fd5b6000825160005b81811015610cc45760208186018101518583015201610caa565b50600092019182525091905056fea26469706673582212207d1a9d88b0ba14ca908470a69ea19a09d2c7617056be2605039bc4d121f4fc4b64736f6c634300081c0033", - "deployedBytecode": "0x608060405234801561001057600080fd5b506004361061007d5760003560e01c806380a38d971161005b57806380a38d9714610128578063b6c2141b14610148578063c7170bb61461015d578063f69c32cd1461016657600080fd5b80630716326d14610082578063474740b1146100e65780634ec42e8e146100fd575b600080fd5b6100bc6100903660046108fc565b60026020819052600091825260409091208054600182015491909201546001600160a01b039092169183565b604080516001600160a01b0390941684526020840192909252908201526060015b60405180910390f35b6100ef60005481565b6040519081526020016100dd565b600154610110906001600160a01b031681565b6040516001600160a01b0390911681526020016100dd565b61013b610136366004610915565b610186565b6040516100dd919061098c565b61015b6101563660046109cf565b6101d3565b005b6100ef60035481565b610179610174366004610a36565b6102c3565b6040516100dd9190610a9d565b60408051600180825281830190925260609160009190602080830190803683370190505090506001816000815181106101c1576101c1610aed565b60209081029190910101529392505050565b60005481908111156101f857604051630d67f41160e21b815260040160405180910390fd5b60005b818110156102bd573684848381811061021657610216610aed565b60600291909101915061024a9050336102326020840184610b28565b6001546001600160a01b03169190602085013561045c565b604080820135600090815260026020522081906102678282610b45565b505060408101356020820180359061027f9084610b28565b6001600160a01b03167ff998960b1c6f0e0e89b7bbe6b6fbf3e03e6f08eee5b8430877d8adb8e149d58060405160405180910390a4506001016101fb565b50505050565b6102cb610838565b60035482146102ed57604051637db491eb60e01b815260040160405180910390fd5b60006102f984846104b6565b905060005b60038110156104545760006002600084846003811061031f5761031f610aed565b602090810291909101516040908101518352828201939093529082016000908120835160608101855281546001600160a01b0316815260018201549381018490526002909101549381018490529350911515919015159085856003811061038857610388610aed565b6020020151602001516001600160a01b031684600001516001600160a01b03161480156103d9575060008686600381106103c4576103c4610aed565b6020020151602001516001600160a01b031614155b9050600060405180606001604052808888600381106103fa576103fa610aed565b60200201516040015181526020018660200151815260200185801561041c5750845b80156104255750835b1515905290508088876003811061043e5761043e610aed565b60200201525050600190930192506102fe915050565b505092915050565b604080516001600160a01b0385811660248301528416604482015260648082018490528251808303909101815260849091019091526020810180516001600160e01b03166323b872dd60e01b1790526102bd908590610691565b6104be610877565b60005b8281101561068a576000600260008686858181106104e1576104e1610aed565b9050610100020160e0013581526020019081526020016000206040518060600160405290816000820160009054906101000a90046001600160a01b03166001600160a01b03166001600160a01b03168152602001600182015481526020016002820154815250509050600260008460006003811061056157610561610aed565b602002015160400151815260200190815260200160002060010154816020015111156105cd576020830180516040850152835190528484838181106105a8576105a8610aed565b905061010002018036038101906105bf9190610beb565b8360005b6020020152610681565b602080840151604090810151600090815260028352206001015490820151111561062c576020830151604084015284848381811061060d5761060d610aed565b905061010002018036038101906106249190610beb565b8360016105c3565b604080840151810151600090815260026020908152919020600101549082015111156106815784848381811061066457610664610aed565b9050610100020180360381019061067b9190610beb565b60408401525b506001016104c1565b5092915050565b60006106a66001600160a01b038416836106fe565b905080516000141580156106cb5750808060200190518101906106c99190610c81565b155b156106f957604051635274afe760e01b81526001600160a01b03841660048201526024015b60405180910390fd5b505050565b606061070c83836000610713565b9392505050565b6060814710156107385760405163cd78605960e01b81523060048201526024016106f0565b600080856001600160a01b031684866040516107549190610ca3565b60006040518083038185875af1925050503d8060008114610791576040519150601f19603f3d011682016040523d82523d6000602084013e610796565b606091505b50915091506107a68683836107b0565b9695505050505050565b6060826107c5576107c08261080c565b61070c565b81511580156107dc57506001600160a01b0384163b155b1561080557604051639996b31560e01b81526001600160a01b03851660048201526024016106f0565b508061070c565b80511561081c5780518082602001fd5b604051630a12f52160e11b815260040160405180910390fd5b50565b60405180606001604052806003905b60408051606081018252600080825260208083018290529282015282526000199092019101816108475790505090565b60405180606001604052806003905b61088e6108a4565b8152602001906001900390816108865790505090565b60405180606001604052806108e86040518060c001604052806000815260200160008152602001600081526020016000815260200160008152602001600081525090565b815260006020820181905260409091015290565b60006020828403121561090e57600080fd5b5035919050565b6000806020838503121561092857600080fd5b823567ffffffffffffffff81111561093f57600080fd5b8301601f8101851361095057600080fd5b803567ffffffffffffffff81111561096757600080fd5b85602060c08302840101111561097c57600080fd5b6020919091019590945092505050565b602080825282518282018190526000918401906040840190835b818110156109c45783518352602093840193909201916001016109a6565b509095945050505050565b600080602083850312156109e257600080fd5b823567ffffffffffffffff8111156109f957600080fd5b8301601f81018513610a0a57600080fd5b803567ffffffffffffffff811115610a2157600080fd5b85602060608302840101111561097c57600080fd5b60008060208385031215610a4957600080fd5b823567ffffffffffffffff811115610a6057600080fd5b8301601f81018513610a7157600080fd5b803567ffffffffffffffff811115610a8857600080fd5b8560208260081b840101111561097c57600080fd5b6101208101818360005b6003811015610ae4578151805184526020810151602085015260408101511515604085015250606083019250602082019150600181019050610aa7565b50505092915050565b634e487b7160e01b600052603260045260246000fd5b6001600160a01b038116811461083557600080fd5b8035610b2381610b03565b919050565b600060208284031215610b3a57600080fd5b813561070c81610b03565b8135610b5081610b03565b81546001600160a01b0319166001600160a01b039190911617815560208201356001820155604090910135600290910155565b6040516060810167ffffffffffffffff81118282101715610bb457634e487b7160e01b600052604160045260246000fd5b60405290565b60405160c0810167ffffffffffffffff81118282101715610bb457634e487b7160e01b600052604160045260246000fd5b600081830361010081128015610c0057600080fd5b506000610c0b610b83565b60c0831215610c18578182fd5b610c20610bba565b853581526020808701359082015260408087013590820152606080870135908201526080808701359082015260a080870135908201528082529250610c6760c08601610b18565b602082015260e09490940135604085015250919392505050565b600060208284031215610c9357600080fd5b8151801515811461070c57600080fd5b6000825160005b81811015610cc45760208186018101518583015201610caa565b50600092019182525091905056fea26469706673582212207d1a9d88b0ba14ca908470a69ea19a09d2c7617056be2605039bc4d121f4fc4b64736f6c634300081c0033", - "linkReferences": {}, - "deployedLinkReferences": {} -} diff --git a/evmlib/artifacts/PaymentVaultNoProxyV2.json b/evmlib/artifacts/PaymentVaultNoProxyV2.json new file mode 100644 index 0000000000..d5e9454faf --- /dev/null +++ b/evmlib/artifacts/PaymentVaultNoProxyV2.json @@ -0,0 +1,383 @@ +{ + "_format": "hh-sol-artifact-1", + "contractName": "PaymentVault", + "sourceName": "contracts/PaymentVaultNoProxyV2.sol", + "abi": [ + { + "inputs": [ + { + "internalType": "contract IERC20", + "name": "_antToken", + "type": "address" + } + ], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "target", + "type": "address" + } + ], + "name": "AddressEmptyCode", + "type": "error" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "account", + "type": "address" + } + ], + "name": "AddressInsufficientBalance", + "type": "error" + }, + { + "inputs": [], + "name": "AntTokenNull", + "type": "error" + }, + { + "inputs": [], + "name": "BatchLimitExceeded", + "type": "error" + }, + { + "inputs": [], + "name": "FailedInnerCall", + "type": "error" + }, + { + "inputs": [], + "name": "InvalidInputLength", + "type": "error" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "token", + "type": "address" + } + ], + "name": "SafeERC20FailedOperation", + "type": "error" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "rewardsAddress", + "type": "address" + }, + { + "indexed": true, + "internalType": "uint256", + "name": "amount", + "type": "uint256" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "quoteHash", + "type": "bytes32" + } + ], + "name": "DataPaymentMade", + "type": "event" + }, + { + "inputs": [], + "name": "antToken", + "outputs": [ + { + "internalType": "contract IERC20", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "batchLimit", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "name": "completedPayments", + "outputs": [ + { + "internalType": "bytes16", + "name": "rewardsAddress", + "type": "bytes16" + }, + { + "internalType": "uint128", + "name": "amount", + "type": "uint128" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "dataSize", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "closeRecordsStored", + "type": "uint256" + }, + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "records", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.Record[]", + "name": "recordsPerType", + "type": "tuple[]" + }, + { + "internalType": "uint256", + "name": "maxRecords", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "receivedPaymentCount", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "liveTime", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "networkDensity", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "networkSize", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.QuotingMetrics[]", + "name": "_metrics", + "type": "tuple[]" + } + ], + "name": "getQuote", + "outputs": [ + { + "internalType": "uint256[]", + "name": "prices", + "type": "uint256[]" + } + ], + "stateMutability": "pure", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "rewardsAddress", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "quoteHash", + "type": "bytes32" + } + ], + "internalType": "struct IPaymentVault.DataPayment[]", + "name": "_payments", + "type": "tuple[]" + } + ], + "name": "payForQuotes", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "requiredPaymentVerificationLength", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "dataSize", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "closeRecordsStored", + "type": "uint256" + }, + { + "components": [ + { + "internalType": "enum IPaymentVault.DataType", + "name": "dataType", + "type": "uint8" + }, + { + "internalType": "uint256", + "name": "records", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.Record[]", + "name": "recordsPerType", + "type": "tuple[]" + }, + { + "internalType": "uint256", + "name": "maxRecords", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "receivedPaymentCount", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "liveTime", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "networkDensity", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "networkSize", + "type": "uint256" + } + ], + "internalType": "struct IPaymentVault.QuotingMetrics", + "name": "metrics", + "type": "tuple" + }, + { + "internalType": "address", + "name": "rewardsAddress", + "type": "address" + }, + { + "internalType": "bytes32", + "name": "quoteHash", + "type": "bytes32" + } + ], + "internalType": "struct IPaymentVault.PaymentVerification[]", + "name": "_payments", + "type": "tuple[]" + } + ], + "name": "verifyPayment", + "outputs": [ + { + "components": [ + { + "internalType": "bytes32", + "name": "quoteHash", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "amountPaid", + "type": "uint256" + }, + { + "internalType": "bool", + "name": "isValid", + "type": "bool" + } + ], + "internalType": "struct IPaymentVault.PaymentVerificationResult[3]", + "name": "verificationResults", + "type": "tuple[3]" + } + ], + "stateMutability": "view", + "type": "function" + } + ], + "bytecode": "0x6080604052348015600f57600080fd5b50604051610f92380380610f92833981016040819052602c916076565b6001600160a01b038116605257604051632d06160b60e21b815260040160405180910390fd5b600080546001600160a01b0319166001600160a01b039290921691909117905560a4565b600060208284031215608757600080fd5b81516001600160a01b0381168114609d57600080fd5b9392505050565b610edf806100b36000396000f3fe608060405234801561001057600080fd5b506004361061007d5760003560e01c80634ec42e8e1161005b5780634ec42e8e146100e2578063b6c2141b1461010d578063c7170bb614610122578063fe3c806e1461012a57600080fd5b806338f03e75146100825780633ffbb252146100ab578063474740b1146100cb575b600080fd5b610095610090366004610a1e565b610188565b6040516100a29190610a60565b60405180910390f35b6100be6100b9366004610a1e565b61034f565b6040516100a29190610ab0565b6100d461020081565b6040519081526020016100a2565b6000546100f5906001600160a01b031681565b6040516001600160a01b0390911681526020016100a2565b61012061011b366004610af3565b6103d8565b005b6100d4600581565b610160610138366004610b6a565b600160205260009081526040902054608081901b90600160801b90046001600160801b031682565b604080516001600160801b031990931683526001600160801b039091166020830152016100a2565b6101906108f7565b600582146101b157604051637db491eb60e01b815260040160405180910390fd5b60006101bd848461051f565b905060005b6003811015610347576000600160008484600381106101e3576101e3610b83565b6020908102919091015160409081015183528282019390935290820160009081208351808501909452546001600160801b0319608082901b168452600160801b90046001600160801b0316918301829052919250600191901580159061025657508183602001516001600160801b031610155b9050600061028b86866003811061026f5761026f610b83565b6020020151602001516001600160801b031960609190911b1690565b84516001600160801b031990811691161480156102cc575060008686600381106102b7576102b7610b83565b6020020151602001516001600160a01b031614155b9050600060405180606001604052808888600381106102ed576102ed610b83565b602002015160400151815260200186602001516001600160801b031681526020018480156103185750835b1515905290508088876003811061033157610331610b83565b60200201525050600190930192506101c2915050565b505092915050565b606060008267ffffffffffffffff81111561036c5761036c610b99565b604051908082528060200260200182016040528015610395578160200160208202803683370190505b50905060005b838110156103d057600060019050808383815181106103bc576103bc610b83565b60209081029190910101525060010161039b565b509392505050565b806102008111156103fc57604051630d67f41160e21b815260040160405180910390fd5b60005b81811015610519573684848381811061041a5761041a610b83565b60600291909101915061044e9050336104366020840184610bcb565b6000546001600160a01b031691906020850135610702565b60408051808201909152806104796104696020850185610bcb565b60601b6001600160801b03191690565b6001600160801b0319168152602083810180356001600160801b03818116948401949094526040808701356000818152600186529190912086519690940151909416600160801b0260809590951c949094179091559091906104db9084610bcb565b6001600160a01b03167ff998960b1c6f0e0e89b7bbe6b6fbf3e03e6f08eee5b8430877d8adb8e149d58060405160405180910390a4506001016103ff565b50505050565b610527610936565b60005b828110156106fb5760006001600086868581811061054a5761054a610b83565b905060200281019061055c9190610be6565b60409081013582526020808301939093529081016000908120825180840184529054608081901b6001600160801b03191682526001600160801b03600160801b918290048116838701908152895186015185526001909652939092205493519094509204811691161115610614576020830180516040850152835190528484838181106105eb576105eb610b83565b90506020028101906105fd9190610be6565b61060690610d66565b8360005b60200201526106f2565b6020808401516040908101516000908152600183522054908201516001600160801b03600160801b909204821691161115610688576020830151604084015284848381811061066557610665610b83565b90506020028101906106779190610be6565b61068090610d66565b83600161060a565b60408084015181015160009081526001602090815291902054908201516001600160801b03600160801b9092048216911611156106f2578484838181106106d1576106d1610b83565b90506020028101906106e39190610be6565b6106ec90610d66565b60408401525b5060010161052a565b5092915050565b604080516001600160a01b038581166024830152848116604483015260648083018590528351808403909101815260849092019092526020810180516001600160e01b03166323b872dd60e01b17905261051991869190600090610768908416836107c0565b9050805160001415801561078d57508080602001905181019061078b9190610e58565b155b156107bb57604051635274afe760e01b81526001600160a01b03841660048201526024015b60405180910390fd5b505050565b60606107ce838360006107d5565b9392505050565b6060814710156107fa5760405163cd78605960e01b81523060048201526024016107b2565b600080856001600160a01b031684866040516108169190610e7a565b60006040518083038185875af1925050503d8060008114610853576040519150601f19603f3d011682016040523d82523d6000602084013e610858565b606091505b5091509150610868868383610872565b9695505050505050565b60608261088757610882826108ce565b6107ce565b815115801561089e57506001600160a01b0384163b155b156108c757604051639996b31560e01b81526001600160a01b03851660048201526024016107b2565b50806107ce565b8051156108de5780518082602001fd5b604051630a12f52160e11b815260040160405180910390fd5b60405180606001604052806003905b60408051606081018252600080825260208083018290529282015282526000199092019101816109065790505090565b60405180606001604052806003905b61094d610963565b8152602001906001900390816109455790505090565b60405180606001604052806109be604080516101208101909152806000815260200160008152602001600081526020016060815260200160008152602001600081526020016000815260200160008152602001600081525090565b815260006020820181905260409091015290565b60008083601f8401126109e457600080fd5b50813567ffffffffffffffff8111156109fc57600080fd5b6020830191508360208260051b8501011115610a1757600080fd5b9250929050565b60008060208385031215610a3157600080fd5b823567ffffffffffffffff811115610a4857600080fd5b610a54858286016109d2565b90969095509350505050565b6101208101818360005b6003811015610aa7578151805184526020810151602085015260408101511515604085015250606083019250602082019150600181019050610a6a565b50505092915050565b602080825282518282018190526000918401906040840190835b81811015610ae8578351835260209384019390920191600101610aca565b509095945050505050565b60008060208385031215610b0657600080fd5b823567ffffffffffffffff811115610b1d57600080fd5b8301601f81018513610b2e57600080fd5b803567ffffffffffffffff811115610b4557600080fd5b856020606083028401011115610b5a57600080fd5b6020919091019590945092505050565b600060208284031215610b7c57600080fd5b5035919050565b634e487b7160e01b600052603260045260246000fd5b634e487b7160e01b600052604160045260246000fd5b80356001600160a01b0381168114610bc657600080fd5b919050565b600060208284031215610bdd57600080fd5b6107ce82610baf565b60008235605e19833603018112610bfc57600080fd5b9190910192915050565b6040805190810167ffffffffffffffff81118282101715610c2957610c29610b99565b60405290565b6040516060810167ffffffffffffffff81118282101715610c2957610c29610b99565b604051610120810167ffffffffffffffff81118282101715610c2957610c29610b99565b604051601f8201601f1916810167ffffffffffffffff81118282101715610c9f57610c9f610b99565b604052919050565b803560048110610bc657600080fd5b600082601f830112610cc757600080fd5b813567ffffffffffffffff811115610ce157610ce1610b99565b610cf060208260051b01610c76565b8082825260208201915060208360061b860101925085831115610d1257600080fd5b602085015b83811015610d5c5760408188031215610d2f57600080fd5b610d37610c06565b610d4082610ca7565b8152602082810135818301529084529290920191604001610d17565b5095945050505050565b600060608236031215610d7857600080fd5b610d80610c2f565b823567ffffffffffffffff811115610d9757600080fd5b8301610120368290031215610dab57600080fd5b610db3610c52565b610dbc82610ca7565b81526020828101359082015260408083013590820152606082013567ffffffffffffffff811115610dec57600080fd5b610df836828501610cb6565b6060830152506080828101359082015260a0808301359082015260c0808301359082015260e0808301359082015261010091820135918101919091528152610e4260208401610baf565b6020820152604092830135928101929092525090565b600060208284031215610e6a57600080fd5b815180151581146107ce57600080fd5b6000825160005b81811015610e9b5760208186018101518583015201610e81565b50600092019182525091905056fea2646970667358221220753415e69ca835f41e469191df2ec65c7858e1fc9c210c5b54c77a49b40d3c9564736f6c634300081c0033", + "deployedBytecode": "0x608060405234801561001057600080fd5b506004361061007d5760003560e01c80634ec42e8e1161005b5780634ec42e8e146100e2578063b6c2141b1461010d578063c7170bb614610122578063fe3c806e1461012a57600080fd5b806338f03e75146100825780633ffbb252146100ab578063474740b1146100cb575b600080fd5b610095610090366004610a1e565b610188565b6040516100a29190610a60565b60405180910390f35b6100be6100b9366004610a1e565b61034f565b6040516100a29190610ab0565b6100d461020081565b6040519081526020016100a2565b6000546100f5906001600160a01b031681565b6040516001600160a01b0390911681526020016100a2565b61012061011b366004610af3565b6103d8565b005b6100d4600581565b610160610138366004610b6a565b600160205260009081526040902054608081901b90600160801b90046001600160801b031682565b604080516001600160801b031990931683526001600160801b039091166020830152016100a2565b6101906108f7565b600582146101b157604051637db491eb60e01b815260040160405180910390fd5b60006101bd848461051f565b905060005b6003811015610347576000600160008484600381106101e3576101e3610b83565b6020908102919091015160409081015183528282019390935290820160009081208351808501909452546001600160801b0319608082901b168452600160801b90046001600160801b0316918301829052919250600191901580159061025657508183602001516001600160801b031610155b9050600061028b86866003811061026f5761026f610b83565b6020020151602001516001600160801b031960609190911b1690565b84516001600160801b031990811691161480156102cc575060008686600381106102b7576102b7610b83565b6020020151602001516001600160a01b031614155b9050600060405180606001604052808888600381106102ed576102ed610b83565b602002015160400151815260200186602001516001600160801b031681526020018480156103185750835b1515905290508088876003811061033157610331610b83565b60200201525050600190930192506101c2915050565b505092915050565b606060008267ffffffffffffffff81111561036c5761036c610b99565b604051908082528060200260200182016040528015610395578160200160208202803683370190505b50905060005b838110156103d057600060019050808383815181106103bc576103bc610b83565b60209081029190910101525060010161039b565b509392505050565b806102008111156103fc57604051630d67f41160e21b815260040160405180910390fd5b60005b81811015610519573684848381811061041a5761041a610b83565b60600291909101915061044e9050336104366020840184610bcb565b6000546001600160a01b031691906020850135610702565b60408051808201909152806104796104696020850185610bcb565b60601b6001600160801b03191690565b6001600160801b0319168152602083810180356001600160801b03818116948401949094526040808701356000818152600186529190912086519690940151909416600160801b0260809590951c949094179091559091906104db9084610bcb565b6001600160a01b03167ff998960b1c6f0e0e89b7bbe6b6fbf3e03e6f08eee5b8430877d8adb8e149d58060405160405180910390a4506001016103ff565b50505050565b610527610936565b60005b828110156106fb5760006001600086868581811061054a5761054a610b83565b905060200281019061055c9190610be6565b60409081013582526020808301939093529081016000908120825180840184529054608081901b6001600160801b03191682526001600160801b03600160801b918290048116838701908152895186015185526001909652939092205493519094509204811691161115610614576020830180516040850152835190528484838181106105eb576105eb610b83565b90506020028101906105fd9190610be6565b61060690610d66565b8360005b60200201526106f2565b6020808401516040908101516000908152600183522054908201516001600160801b03600160801b909204821691161115610688576020830151604084015284848381811061066557610665610b83565b90506020028101906106779190610be6565b61068090610d66565b83600161060a565b60408084015181015160009081526001602090815291902054908201516001600160801b03600160801b9092048216911611156106f2578484838181106106d1576106d1610b83565b90506020028101906106e39190610be6565b6106ec90610d66565b60408401525b5060010161052a565b5092915050565b604080516001600160a01b038581166024830152848116604483015260648083018590528351808403909101815260849092019092526020810180516001600160e01b03166323b872dd60e01b17905261051991869190600090610768908416836107c0565b9050805160001415801561078d57508080602001905181019061078b9190610e58565b155b156107bb57604051635274afe760e01b81526001600160a01b03841660048201526024015b60405180910390fd5b505050565b60606107ce838360006107d5565b9392505050565b6060814710156107fa5760405163cd78605960e01b81523060048201526024016107b2565b600080856001600160a01b031684866040516108169190610e7a565b60006040518083038185875af1925050503d8060008114610853576040519150601f19603f3d011682016040523d82523d6000602084013e610858565b606091505b5091509150610868868383610872565b9695505050505050565b60608261088757610882826108ce565b6107ce565b815115801561089e57506001600160a01b0384163b155b156108c757604051639996b31560e01b81526001600160a01b03851660048201526024016107b2565b50806107ce565b8051156108de5780518082602001fd5b604051630a12f52160e11b815260040160405180910390fd5b60405180606001604052806003905b60408051606081018252600080825260208083018290529282015282526000199092019101816109065790505090565b60405180606001604052806003905b61094d610963565b8152602001906001900390816109455790505090565b60405180606001604052806109be604080516101208101909152806000815260200160008152602001600081526020016060815260200160008152602001600081526020016000815260200160008152602001600081525090565b815260006020820181905260409091015290565b60008083601f8401126109e457600080fd5b50813567ffffffffffffffff8111156109fc57600080fd5b6020830191508360208260051b8501011115610a1757600080fd5b9250929050565b60008060208385031215610a3157600080fd5b823567ffffffffffffffff811115610a4857600080fd5b610a54858286016109d2565b90969095509350505050565b6101208101818360005b6003811015610aa7578151805184526020810151602085015260408101511515604085015250606083019250602082019150600181019050610a6a565b50505092915050565b602080825282518282018190526000918401906040840190835b81811015610ae8578351835260209384019390920191600101610aca565b509095945050505050565b60008060208385031215610b0657600080fd5b823567ffffffffffffffff811115610b1d57600080fd5b8301601f81018513610b2e57600080fd5b803567ffffffffffffffff811115610b4557600080fd5b856020606083028401011115610b5a57600080fd5b6020919091019590945092505050565b600060208284031215610b7c57600080fd5b5035919050565b634e487b7160e01b600052603260045260246000fd5b634e487b7160e01b600052604160045260246000fd5b80356001600160a01b0381168114610bc657600080fd5b919050565b600060208284031215610bdd57600080fd5b6107ce82610baf565b60008235605e19833603018112610bfc57600080fd5b9190910192915050565b6040805190810167ffffffffffffffff81118282101715610c2957610c29610b99565b60405290565b6040516060810167ffffffffffffffff81118282101715610c2957610c29610b99565b604051610120810167ffffffffffffffff81118282101715610c2957610c29610b99565b604051601f8201601f1916810167ffffffffffffffff81118282101715610c9f57610c9f610b99565b604052919050565b803560048110610bc657600080fd5b600082601f830112610cc757600080fd5b813567ffffffffffffffff811115610ce157610ce1610b99565b610cf060208260051b01610c76565b8082825260208201915060208360061b860101925085831115610d1257600080fd5b602085015b83811015610d5c5760408188031215610d2f57600080fd5b610d37610c06565b610d4082610ca7565b8152602082810135818301529084529290920191604001610d17565b5095945050505050565b600060608236031215610d7857600080fd5b610d80610c2f565b823567ffffffffffffffff811115610d9757600080fd5b8301610120368290031215610dab57600080fd5b610db3610c52565b610dbc82610ca7565b81526020828101359082015260408083013590820152606082013567ffffffffffffffff811115610dec57600080fd5b610df836828501610cb6565b6060830152506080828101359082015260a0808301359082015260c0808301359082015260e0808301359082015261010091820135918101919091528152610e4260208401610baf565b6020820152604092830135928101929092525090565b600060208284031215610e6a57600080fd5b815180151581146107ce57600080fd5b6000825160005b81811015610e9b5760208186018101518583015201610e81565b50600092019182525091905056fea2646970667358221220753415e69ca835f41e469191df2ec65c7858e1fc9c210c5b54c77a49b40d3c9564736f6c634300081c0033", + "linkReferences": {}, + "deployedLinkReferences": {} +} diff --git a/evmlib/src/contract/data_payments/mod.rs b/evmlib/src/contract/data_payments/mod.rs deleted file mode 100644 index 45a4f981a3..0000000000 --- a/evmlib/src/contract/data_payments/mod.rs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -pub mod error; - -use crate::common; -use crate::common::{Address, Calldata, TxHash}; -use crate::contract::data_payments::error::Error; -use crate::contract::data_payments::DataPaymentsContract::DataPaymentsContractInstance; -use alloy::network::TransactionBuilder; -use alloy::providers::{Network, Provider}; -use alloy::sol; -use alloy::transports::Transport; - -/// The max amount of transfers within one data payments transaction. -pub const MAX_TRANSFERS_PER_TRANSACTION: usize = 512; - -sol!( - #[allow(clippy::too_many_arguments)] - #[allow(missing_docs)] - #[sol(rpc)] - DataPaymentsContract, - "artifacts/DataPayments.json" -); - -pub struct DataPaymentsHandler, N: Network> { - pub contract: DataPaymentsContractInstance, -} - -impl DataPaymentsHandler -where - T: Transport + Clone, - P: Provider, - N: Network, -{ - /// Create a new ChunkPayments contract instance. - pub fn new(contract_address: Address, provider: P) -> Self { - let contract = DataPaymentsContract::new(contract_address, provider); - DataPaymentsHandler { contract } - } - - /// Deploys the ChunkPayments smart contract to the network of the provider. - /// ONLY DO THIS IF YOU KNOW WHAT YOU ARE DOING! - pub async fn deploy(provider: P, payment_token_address: Address) -> Self { - let contract = DataPaymentsContract::deploy(provider, payment_token_address) - .await - .expect("Could not deploy contract"); - debug!( - "DataPayments contract deployed at: {:?}", - contract.address() - ); - - DataPaymentsHandler { contract } - } - - pub fn set_provider(&mut self, provider: P) { - let address = *self.contract.address(); - self.contract = DataPaymentsContract::new(address, provider); - } - - /// Pay for quotes. - /// Input: (quote_id, reward_address, amount). - pub async fn pay_for_quotes>( - &self, - data_payments: I, - ) -> Result { - let (calldata, to) = self.pay_for_quotes_calldata(data_payments)?; - debug!("Data payments calldata is processed to the address {to:?}"); - - let transaction_request = self - .contract - .provider() - .transaction_request() - .with_to(to) - .with_input(calldata); - - let tx_hash = self - .contract - .provider() - .send_transaction(transaction_request) - .await? - .watch() - .await?; - debug!("Data payments transaction hash: {:?}", tx_hash); - Ok(tx_hash) - } - - /// Pay for quotes. - /// Input: (quote_id, reward_address, amount). - /// Returns the transaction calldata. - pub fn pay_for_quotes_calldata>( - &self, - data_payments: I, - ) -> Result<(Calldata, Address), Error> { - let data_payments: Vec = data_payments - .into_iter() - .map(|(hash, addr, amount)| DataPayments::DataPayment { - rewardsAddress: addr, - amount, - quoteHash: hash, - }) - .collect(); - - if data_payments.len() > MAX_TRANSFERS_PER_TRANSACTION { - return Err(Error::TransferLimitExceeded); - } - - let calldata = self - .contract - .submitDataPayments(data_payments) - .calldata() - .to_owned(); - - Ok((calldata, *self.contract.address())) - } -} diff --git a/evmlib/src/contract/network_token.rs b/evmlib/src/contract/network_token.rs index df18b37a1d..47f11946a1 100644 --- a/evmlib/src/contract/network_token.rs +++ b/evmlib/src/contract/network_token.rs @@ -8,8 +8,7 @@ use crate::common::{Address, Calldata, TxHash, U256}; use crate::contract::network_token::NetworkTokenContract::NetworkTokenContractInstance; -use crate::TX_TIMEOUT; -use alloy::network::TransactionBuilder; +use crate::retry::{retry, send_transaction_with_retries}; use alloy::providers::{Network, Provider}; use alloy::sol; use alloy::transports::{RpcError, Transport, TransportErrorKind}; @@ -30,6 +29,8 @@ pub enum Error { RpcError(#[from] RpcError), #[error(transparent)] PendingTransactionError(#[from] alloy::providers::PendingTransactionError), + #[error("Timeout: {0:?}")] + Timeout(#[from] tokio::time::error::Elapsed), } pub struct NetworkToken, N: Network> { @@ -65,66 +66,36 @@ where /// Get the raw token balance of an address. pub async fn balance_of(&self, account: Address) -> Result { debug!("Getting balance of account: {account:?}"); - let balance = self - .contract - .balanceOf(account) - .call() - .await - .inspect_err(|err| error!("Error getting balance of account: {err:?}"))? - ._0; - debug!("Balance of account: {account} is {balance}"); + let balance = retry( + || async { self.contract.balanceOf(account).call().await }, + "balanceOf", + None, + ) + .await? + ._0; + debug!("Balance of account {account} is {balance}"); Ok(balance) } /// See how many tokens are approved to be spent. pub async fn allowance(&self, owner: Address, spender: Address) -> Result { - debug!("Getting allowance of owner: {owner} for spender: {spender}",); - let balance = self - .contract - .allowance(owner, spender) - .call() - .await - .inspect_err(|err| error!("Error getting allowance: {err:?}"))? - ._0; - debug!("Allowance of owner: {owner} for spender: {spender} is: {balance}"); - Ok(balance) + debug!("Getting allowance of owner: {owner} for spender: {spender}"); + let allowance = retry( + || async { self.contract.allowance(owner, spender).call().await }, + "allowance", + None, + ) + .await? + ._0; + debug!("Allowance of owner: {owner} for spender: {spender} is: {allowance}"); + Ok(allowance) } /// Approve spender to spend a raw amount of tokens. pub async fn approve(&self, spender: Address, value: U256) -> Result { - debug!("Approving spender to spend raw amt of tokens: {value}"); + debug!("Approving spender {spender:?} to spend {value}"); let (calldata, to) = self.approve_calldata(spender, value); - - let transaction_request = self - .contract - .provider() - .transaction_request() - .with_to(to) - .with_input(calldata); - - let pending_tx_builder = self - .contract - .provider() - .send_transaction(transaction_request) - .await - .inspect_err(|err| { - error!( - "Error to send_transaction while approving spender {spender:?} to spend raw amt of tokens {value}: {err:?}" - ) - })? - .with_timeout(Some(TX_TIMEOUT)); - - let pending_tx_hash = *pending_tx_builder.tx_hash(); - - debug!("The approval from sender {spender:?} is pending with tx_hash: {pending_tx_hash:?}",); - - let tx_hash = pending_tx_builder.watch().await.inspect_err(|err| { - error!("Error watching approve tx with hash {pending_tx_hash:?}: {err:?}") - })?; - - debug!("Approve tx with hash {tx_hash:?} is successful"); - - Ok(tx_hash) + send_transaction_with_retries(self.contract.provider(), calldata, to, "approve").await } /// Approve spender to spend a raw amount of tokens. @@ -136,37 +107,9 @@ where /// Transfer a raw amount of tokens. pub async fn transfer(&self, receiver: Address, amount: U256) -> Result { - debug!("Transferring raw amt of tokens: {amount} to {receiver:?}"); + debug!("Transferring raw amount of tokens: {amount} to {receiver:?}"); let (calldata, to) = self.transfer_calldata(receiver, amount); - - let transaction_request = self - .contract - .provider() - .transaction_request() - .with_to(to) - .with_input(calldata); - - let pending_tx_builder = self - .contract - .provider() - .send_transaction(transaction_request) - .await - .inspect_err(|err| { - error!("Error to send_transaction during transfer raw amt of tokens to {receiver:?}: {err:?}") - })? - .with_timeout(Some(TX_TIMEOUT)); - - let pending_tx_hash = *pending_tx_builder.tx_hash(); - debug!( - "The transfer to receiver {receiver:?} is pending with tx_hash: {pending_tx_hash:?}" - ); - let tx_hash = pending_tx_builder.watch().await.inspect_err(|err| { - error!("Error watching transfer tx with hash {pending_tx_hash:?}: {err:?}") - })?; - - debug!("Transfer tx with hash {tx_hash:?} is successful"); - - Ok(tx_hash) + send_transaction_with_retries(self.contract.provider(), calldata, to, "transfer").await } /// Transfer a raw amount of tokens. diff --git a/evmlib/src/contract/payment_vault/error.rs b/evmlib/src/contract/payment_vault/error.rs index f4a5b76cce..31f24e794f 100644 --- a/evmlib/src/contract/payment_vault/error.rs +++ b/evmlib/src/contract/payment_vault/error.rs @@ -12,4 +12,6 @@ pub enum Error { PaymentInvalid, #[error("Payment verification length must be 3.")] PaymentVerificationLengthInvalid, + #[error("Timeout: {0:?}")] + Timeout(#[from] tokio::time::error::Elapsed), } diff --git a/evmlib/src/contract/payment_vault/handler.rs b/evmlib/src/contract/payment_vault/handler.rs index 4330f31fbe..026ec37c72 100644 --- a/evmlib/src/contract/payment_vault/handler.rs +++ b/evmlib/src/contract/payment_vault/handler.rs @@ -2,8 +2,8 @@ use crate::common::{Address, Amount, Calldata, TxHash}; use crate::contract::payment_vault::error::Error; use crate::contract::payment_vault::interface::IPaymentVault; use crate::contract::payment_vault::interface::IPaymentVault::IPaymentVaultInstance; -use crate::TX_TIMEOUT; -use alloy::network::{Network, TransactionBuilder}; +use crate::retry::{retry, send_transaction_with_retries}; +use alloy::network::Network; use alloy::providers::Provider; use alloy::transports::Transport; @@ -35,14 +35,25 @@ where metrics: I, ) -> Result, Error> { let metrics: Vec<_> = metrics.into_iter().map(|v| v.into()).collect(); - let mut amounts = self.contract.getQuote(metrics.clone()).call().await?.prices; - // FIXME: temporary logic until the smart contract gets updated + debug!("Getting quotes for metrics: {metrics:?}"); + + let mut amounts = retry( + || async { self.contract.getQuote(metrics.clone()).call().await }, + "getQuote", + None, + ) + .await? + .prices; + + // FIXME: temporary logic until the local smart contract gets updated if amounts.len() == 1 { let value = amounts[0]; amounts.resize(metrics.len(), value); } + debug!("Returned quotes are: {:?}", amounts); + Ok(amounts) } @@ -51,31 +62,10 @@ where &self, data_payments: I, ) -> Result { + debug!("Paying for quotes."); let (calldata, to) = self.pay_for_quotes_calldata(data_payments)?; - - let transaction_request = self - .contract - .provider() - .transaction_request() - .with_to(to) - .with_input(calldata); - - let pending_tx_builder = self - .contract - .provider() - .send_transaction(transaction_request) + send_transaction_with_retries(self.contract.provider(), calldata, to, "pay for quotes") .await - .inspect_err(|err| error!("Error to send_transaction during pay_for_quotes: {err:?}"))? - .with_timeout(Some(TX_TIMEOUT)); - - let pending_tx_hash = pending_tx_builder.tx_hash(); - debug!("pay_for_quotes is pending with tx hash: {pending_tx_hash}"); - - let tx_hash = pending_tx_builder.watch().await.inspect_err(|err| { - error!("Error to watch transaction during pay_for_quotes: {err:?}") - })?; - - Ok(tx_hash) } /// Returns the pay for quotes transaction calldata. @@ -105,12 +95,22 @@ where .map(|v| v.into()) .collect(); - let results = self - .contract - .verifyPayment(payment_verifications) - .call() - .await? - .verificationResults; + debug!("Verifying payments: {payment_verifications:?}"); + + let results = retry( + || async { + self.contract + .verifyPayment(payment_verifications.clone()) + .call() + .await + }, + "verifyPayment", + None, + ) + .await? + .verificationResults; + + debug!("Payment verification results: {:?}", results); Ok(results) } diff --git a/evmlib/src/contract/payment_vault/implementation.rs b/evmlib/src/contract/payment_vault/implementation.rs index 64fd9da1f9..3ac3086160 100644 --- a/evmlib/src/contract/payment_vault/implementation.rs +++ b/evmlib/src/contract/payment_vault/implementation.rs @@ -1,4 +1,4 @@ -use crate::common::{Address, U256}; +use crate::common::Address; use alloy::network::Network; use alloy::providers::Provider; use alloy::sol; @@ -8,21 +8,17 @@ sol!( #[allow(missing_docs)] #[sol(rpc)] PaymentVaultImplementation, - "artifacts/PaymentVaultNoProxy.json" + "artifacts/PaymentVaultNoProxyV2.json" ); /// Deploys the payment vault contract and returns the contract address -pub async fn deploy( - provider: &P, - network_token_address: Address, - batch_limit: U256, -) -> Address +pub async fn deploy(provider: &P, network_token_address: Address) -> Address where T: Transport + Clone, P: Provider, N: Network, { - let contract = PaymentVaultImplementation::deploy(provider, network_token_address, batch_limit) + let contract = PaymentVaultImplementation::deploy(provider, network_token_address) .await .expect("Could not deploy payment vault implementation contract"); diff --git a/evmlib/src/contract/payment_vault/interface.rs b/evmlib/src/contract/payment_vault/interface.rs index 1e2e0f1e7c..6d4b930146 100644 --- a/evmlib/src/contract/payment_vault/interface.rs +++ b/evmlib/src/contract/payment_vault/interface.rs @@ -8,7 +8,7 @@ sol!( #[derive(Debug)] #[sol(rpc)] IPaymentVault, - "abi/IPaymentVault.json" + "abi/IPaymentVaultV2.json" ); impl From<(QuoteHash, QuotingMetrics, Address)> for IPaymentVault::PaymentVerification { @@ -34,7 +34,17 @@ impl From<(QuoteHash, Address, Amount)> for IPaymentVault::DataPayment { impl From for IPaymentVault::QuotingMetrics { fn from(value: QuotingMetrics) -> Self { Self { + dataType: data_type_conversion(value.data_type), + dataSize: U256::from(value.data_size), closeRecordsStored: U256::from(value.close_records_stored), + recordsPerType: value + .records_per_type + .into_iter() + .map(|(data_type, amount)| IPaymentVault::Record { + dataType: data_type_conversion(data_type), + records: U256::from(amount), + }) + .collect(), maxRecords: U256::from(value.max_records), receivedPaymentCount: U256::from(value.received_payment_count), liveTime: U256::from(value.live_time), @@ -44,3 +54,13 @@ impl From for IPaymentVault::QuotingMetrics { } } } + +fn data_type_conversion(data_type: u32) -> u8 { + match data_type { + 0 => 2, // Chunk + 1 => 0, // GraphEntry + 2 => 3, // Pointer + 3 => 1, // Scratchpad + _ => 4, // Does not exist + } +} diff --git a/evmlib/src/lib.rs b/evmlib/src/lib.rs index e2715c0ed6..25c1e58c73 100644 --- a/evmlib/src/lib.rs +++ b/evmlib/src/lib.rs @@ -7,6 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::common::Address; +use crate::utils::get_evm_network; use alloy::primitives::address; use alloy::transports::http::reqwest; use serde::{Deserialize, Serialize}; @@ -23,10 +24,14 @@ pub mod cryptography; #[cfg(feature = "external-signer")] pub mod external_signer; pub mod quoting_metrics; +mod retry; pub mod testnet; pub mod utils; pub mod wallet; +/// Timeout for transactions +const TX_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(24); // Should differ per chain + static PUBLIC_ARBITRUM_ONE_HTTP_RPC_URL: LazyLock = LazyLock::new(|| { "https://arb1.arbitrum.io/rpc" .parse() @@ -40,20 +45,22 @@ static PUBLIC_ARBITRUM_SEPOLIA_HTTP_RPC_URL: LazyLock = LazyLock:: }); const ARBITRUM_ONE_PAYMENT_TOKEN_ADDRESS: Address = - address!("4bc1aCE0E66170375462cB4E6Af42Ad4D5EC689C"); + address!("0xa78d8321B20c4Ef90eCd72f2588AA985A4BDb684"); const ARBITRUM_SEPOLIA_PAYMENT_TOKEN_ADDRESS: Address = address!("BE1802c27C324a28aeBcd7eeC7D734246C807194"); -// Should be updated when the smart contract changes! +const ARBITRUM_SEPOLIA_TEST_PAYMENT_TOKEN_ADDRESS: Address = + address!("4bc1aCE0E66170375462cB4E6Af42Ad4D5EC689C"); + const ARBITRUM_ONE_DATA_PAYMENTS_ADDRESS: Address = - address!("607483B50C5F06c25cDC316b6d1E071084EeC9f5"); + address!("B1b5219f8Aaa18037A2506626Dd0406a46f70BcC"); const ARBITRUM_SEPOLIA_DATA_PAYMENTS_ADDRESS: Address = address!("993C7739f50899A997fEF20860554b8a28113634"); -/// Timeout for transactions -const TX_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +const ARBITRUM_SEPOLIA_TEST_DATA_PAYMENTS_ADDRESS: Address = + address!("7f0842a78f7d4085d975ba91d630d680f91b1295"); #[serde_as] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -81,6 +88,7 @@ pub enum Network { #[default] ArbitrumOne, ArbitrumSepolia, + ArbitrumSepoliaTest, Custom(CustomNetwork), } @@ -89,12 +97,19 @@ impl std::fmt::Display for Network { match self { Network::ArbitrumOne => write!(f, "evm-arbitrum-one"), Network::ArbitrumSepolia => write!(f, "evm-arbitrum-sepolia"), + Network::ArbitrumSepoliaTest => write!(f, "evm-arbitrum-sepolia-test"), Network::Custom(_) => write!(f, "evm-custom"), } } } impl Network { + pub fn new(local: bool) -> Result { + get_evm_network(local).inspect_err(|err| { + warn!("Failed to select EVM network from ENV: {err}"); + }) + } + pub fn new_custom(rpc_url: &str, payment_token_addr: &str, chunk_payments_addr: &str) -> Self { Self::Custom(CustomNetwork::new( rpc_url, @@ -107,6 +122,7 @@ impl Network { match self { Network::ArbitrumOne => "arbitrum-one", Network::ArbitrumSepolia => "arbitrum-sepolia", + Network::ArbitrumSepoliaTest => "arbitrum-sepolia-test", Network::Custom(_) => "custom", } } @@ -115,6 +131,7 @@ impl Network { match self { Network::ArbitrumOne => &PUBLIC_ARBITRUM_ONE_HTTP_RPC_URL, Network::ArbitrumSepolia => &PUBLIC_ARBITRUM_SEPOLIA_HTTP_RPC_URL, + Network::ArbitrumSepoliaTest => &PUBLIC_ARBITRUM_SEPOLIA_HTTP_RPC_URL, Network::Custom(custom) => &custom.rpc_url_http, } } @@ -123,6 +140,7 @@ impl Network { match self { Network::ArbitrumOne => &ARBITRUM_ONE_PAYMENT_TOKEN_ADDRESS, Network::ArbitrumSepolia => &ARBITRUM_SEPOLIA_PAYMENT_TOKEN_ADDRESS, + Network::ArbitrumSepoliaTest => &ARBITRUM_SEPOLIA_TEST_PAYMENT_TOKEN_ADDRESS, Network::Custom(custom) => &custom.payment_token_address, } } @@ -131,6 +149,7 @@ impl Network { match self { Network::ArbitrumOne => &ARBITRUM_ONE_DATA_PAYMENTS_ADDRESS, Network::ArbitrumSepolia => &ARBITRUM_SEPOLIA_DATA_PAYMENTS_ADDRESS, + Network::ArbitrumSepoliaTest => &ARBITRUM_SEPOLIA_TEST_DATA_PAYMENTS_ADDRESS, Network::Custom(custom) => &custom.data_payments_address, } } diff --git a/evmlib/src/quoting_metrics.rs b/evmlib/src/quoting_metrics.rs index c4971a1b03..be383042b9 100644 --- a/evmlib/src/quoting_metrics.rs +++ b/evmlib/src/quoting_metrics.rs @@ -13,8 +13,14 @@ use std::fmt::{Debug, Formatter, Result as FmtResult}; /// Quoting metrics used to generate a quote, or to track peer's status. #[derive(Clone, Eq, PartialEq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct QuotingMetrics { + /// DataTypes presented as its `index` + pub data_type: u32, + /// data size of the record + pub data_size: usize, /// the records stored pub close_records_stored: usize, + /// each entry to be `(data_type_index, num_of_records_of_that_type)` + pub records_per_type: Vec<(u32, u32)>, /// the max_records configured pub max_records: usize, /// number of times that got paid @@ -28,31 +34,11 @@ pub struct QuotingMetrics { pub network_size: Option, } -impl QuotingMetrics { - /// construct an empty QuotingMetrics - pub fn new() -> Self { - Self { - close_records_stored: 0, - max_records: 0, - received_payment_count: 0, - live_time: 0, - network_density: None, - network_size: None, - } - } -} - -impl Default for QuotingMetrics { - fn default() -> Self { - Self::new() - } -} - impl Debug for QuotingMetrics { fn fmt(&self, formatter: &mut Formatter) -> FmtResult { let density_u256 = self.network_density.map(U256::from_be_bytes); - write!(formatter, "QuotingMetrics {{ close_records_stored: {}, max_records: {}, received_payment_count: {}, live_time: {}, network_density: {density_u256:?}, network_size: {:?} }}", - self.close_records_stored, self.max_records, self.received_payment_count, self.live_time, self.network_size) + write!(formatter, "QuotingMetrics {{ data_type: {}, data_size: {}, close_records_stored: {}, records_per_type {:?}, max_records: {}, received_payment_count: {}, live_time: {}, network_density: {density_u256:?}, network_size: {:?} }}", + self.data_type, self.data_size, self.close_records_stored, self.records_per_type, self.max_records, self.received_payment_count, self.live_time, self.network_size) } } diff --git a/evmlib/src/retry.rs b/evmlib/src/retry.rs new file mode 100644 index 0000000000..d802a9f087 --- /dev/null +++ b/evmlib/src/retry.rs @@ -0,0 +1,176 @@ +use crate::common::{Address, Calldata, TxHash}; +use crate::TX_TIMEOUT; +use alloy::network::{Network, TransactionBuilder}; +use alloy::providers::{PendingTransactionBuilder, Provider}; +use alloy::transports::Transport; +use std::time::Duration; + +pub(crate) const MAX_RETRIES: u8 = 3; +const DEFAULT_RETRY_INTERVAL_MS: u64 = 4000; +const BROADCAST_TRANSACTION_TIMEOUT_MS: u64 = 5000; +const WATCH_TIMEOUT_MS: u64 = 1000; + +/// Execute an async closure that returns a result. Retry on failure. +pub(crate) async fn retry( + mut action: F, + operation_id: &str, + retry_interval_ms: Option, +) -> Result +where + F: FnMut() -> Fut + Send, + Fut: std::future::Future> + Send, + E: std::fmt::Debug, +{ + let mut retries = 0; + + loop { + match action().await { + Ok(result) => return Ok(result), + Err(err) => { + if retries == MAX_RETRIES { + error!("{operation_id} failed after {retries} retries: {err:?}"); + return Err(err); + } + + retries += 1; + let retry_interval_ms = retry_interval_ms.unwrap_or(DEFAULT_RETRY_INTERVAL_MS); + let delay = Duration::from_millis(retry_interval_ms * retries.pow(2) as u64); + + warn!( + "Error trying {operation_id}: {err:?}. Retry #{retries} in {:?} second(s).", + delay.as_secs() + ); + + tokio::time::sleep(delay).await; + } + } + } +} + +/// Generic function to send a transaction with retries. +pub(crate) async fn send_transaction_with_retries( + provider: &P, + calldata: Calldata, + to: Address, + tx_identifier: &str, +) -> Result +where + T: Transport + Clone, + P: Provider, + N: Network, + E: From> + + From + + From, +{ + let mut nonce: Option = None; + let mut retries = 0; + + loop { + let mut transaction_request = provider + .transaction_request() + .with_to(to) + .with_input(calldata.clone()); + + // Retry with the same nonce to replace a stuck transaction + if let Some(nonce) = nonce { + transaction_request.set_nonce(nonce); + } else { + nonce = transaction_request.nonce(); + } + + let pending_tx_builder_result = tokio::time::timeout( + Duration::from_millis(BROADCAST_TRANSACTION_TIMEOUT_MS), + provider.send_transaction(transaction_request.clone()), + ) + .await; + + let pending_tx_builder = match pending_tx_builder_result { + Ok(Ok(pending_tx_builder)) => pending_tx_builder, + Ok(Err(err)) => { + if retries == MAX_RETRIES { + error!("Failed to send {tx_identifier} transaction after {retries} retries. Giving up. Error: {err:?}"); + break Err(E::from(err)); + } + + retries += 1; + let retry_interval_ms = DEFAULT_RETRY_INTERVAL_MS; + let delay = Duration::from_millis(retry_interval_ms * retries.pow(2) as u64); + + warn!( + "Error sending {tx_identifier} transaction: {err:?}. Retry #{} in {} second(s).", + retries, + delay.as_secs(), + ); + + tokio::time::sleep(delay).await; + + continue; + } + Err(err) => { + if retries == MAX_RETRIES { + error!("Failed to send {tx_identifier} transaction after {retries} retries. Giving up. Error: {err:?}"); + break Err(E::from(err)); + } + + retries += 1; + let retry_interval_ms = DEFAULT_RETRY_INTERVAL_MS; + let delay = Duration::from_millis(retry_interval_ms * retries.pow(2) as u64); + + warn!( + "Error sending {tx_identifier} transaction: {err:?}. Retry #{} in {} second(s).", + retries, + delay.as_secs(), + ); + + tokio::time::sleep(delay).await; + + continue; + } + }; + + debug!( + "{tx_identifier} transaction is pending with tx_hash: {:?}", + pending_tx_builder.tx_hash() + ); + + let watch_result = retry( + || async { + PendingTransactionBuilder::from_config( + provider.root().clone(), + pending_tx_builder.inner().clone(), + ) + .with_timeout(Some(TX_TIMEOUT)) + .watch() + .await + }, + "watching pending transaction", + Some(WATCH_TIMEOUT_MS), + ) + .await; + + match watch_result { + Ok(tx_hash) => { + debug!("{tx_identifier} transaction with hash {tx_hash:?} is successful"); + break Ok(tx_hash); + } + Err(err) => { + if retries == MAX_RETRIES { + error!("Failed to confirm {tx_identifier} transaction after {retries} retries. Giving up. Error: {err:?}"); + break Err(E::from(err)); + } + + retries += 1; + let retry_interval_ms = DEFAULT_RETRY_INTERVAL_MS; + let delay = Duration::from_millis(retry_interval_ms * retries.pow(2) as u64); + + warn!( + "Error confirming {tx_identifier} transaction: {err:?}. Retry #{} in {} second(s).", + retries, + delay.as_secs(), + ); + + tokio::time::sleep(delay).await; + } + } + } +} diff --git a/evmlib/src/testnet.rs b/evmlib/src/testnet.rs index d9c25bcffd..f47079fe82 100644 --- a/evmlib/src/testnet.rs +++ b/evmlib/src/testnet.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::common::{Address, Amount}; +use crate::common::Address; use crate::contract::network_token::NetworkToken; use crate::contract::payment_vault; use crate::contract::payment_vault::handler::PaymentVaultHandler; @@ -22,8 +22,6 @@ use alloy::providers::{Identity, ProviderBuilder, ReqwestProvider}; use alloy::signers::local::PrivateKeySigner; use alloy::transports::http::{Client, Http}; -const BATCH_LIMIT: u16 = 256; - pub struct Testnet { anvil: AnvilInstance, rpc_url: Url, @@ -150,8 +148,7 @@ pub async fn deploy_data_payments_contract( // Deploy the contract. let payment_vault_contract_address = - payment_vault::implementation::deploy(&provider, token_address, Amount::from(BATCH_LIMIT)) - .await; + payment_vault::implementation::deploy(&provider, token_address).await; // Create a handler for the deployed contract PaymentVaultHandler::new(payment_vault_contract_address, provider) diff --git a/evmlib/src/utils.rs b/evmlib/src/utils.rs index 4e3133713f..40c92ca648 100644 --- a/evmlib/src/utils.rs +++ b/evmlib/src/utils.rs @@ -47,6 +47,34 @@ pub fn dummy_hash() -> Hash { Hash::new(rand::rngs::OsRng.gen()) } +use std::sync::OnceLock; + +static EVM_NETWORK: OnceLock = OnceLock::new(); + +/// Initialize the EVM Network parameters from environment variables or local CSV file. +/// +/// It will first try to get the network from the environment variables. +/// If it fails and `local` is true, it will try to get the network from the local CSV file. +/// If both fail, it will return the default network. +pub fn get_evm_network(local: bool) -> Result { + if let Some(network) = EVM_NETWORK.get() { + return Ok(network.clone()); + } + + let res = match get_evm_network_from_env() { + Ok(evm_network) => Ok(evm_network), + Err(_) if local => Ok(local_evm_network_from_csv() + .map_err(|e| Error::FailedToGetEvmNetwork(e.to_string()))?), + Err(_) => Ok(Network::default()), + }; + + if let Ok(network) = res.as_ref() { + let _ = EVM_NETWORK.set(network.clone()); + } + + res +} + pub fn get_evm_testnet_csv_path() -> Result { let file = data_dir() .ok_or(Error::FailedToGetEvmNetwork( @@ -57,23 +85,10 @@ pub fn get_evm_testnet_csv_path() -> Result { Ok(file) } -/// Create a custom `Network` from the given values -pub fn get_evm_network( - rpc_url: &str, - payment_token_address: &str, - data_payments_address: &str, -) -> Network { - Network::Custom(CustomNetwork::new( - rpc_url, - payment_token_address, - data_payments_address, - )) -} - /// Get the `Network` from environment variables. /// /// Returns an error if we cannot obtain the network from any means. -pub fn get_evm_network_from_env() -> Result { +fn get_evm_network_from_env() -> Result { let evm_vars = [ env::var(RPC_URL) .ok() @@ -93,16 +108,12 @@ pub fn get_evm_network_from_env() -> Result { }) .collect::, Error>>(); - let mut use_local_evm = std::env::var("EVM_NETWORK") + let use_local_evm = std::env::var("EVM_NETWORK") .map(|v| v == "local") .unwrap_or(false); if use_local_evm { info!("Using local EVM network as EVM_NETWORK is set to 'local'"); } - if cfg!(feature = "local") { - use_local_evm = true; - info!("Using local EVM network as 'local' feature flag is enabled"); - } let use_arbitrum_one = std::env::var("EVM_NETWORK") .map(|v| v == "arbitrum-one") @@ -112,12 +123,19 @@ pub fn get_evm_network_from_env() -> Result { .map(|v| v == "arbitrum-sepolia") .unwrap_or(false); + let use_arbitrum_sepolia_test = std::env::var("EVM_NETWORK") + .map(|v| v == "arbitrum-sepolia-test") + .unwrap_or(false); + if use_arbitrum_one { info!("Using Arbitrum One EVM network as EVM_NETWORK is set to 'arbitrum-one'"); Ok(Network::ArbitrumOne) } else if use_arbitrum_sepolia { info!("Using Arbitrum Sepolia EVM network as EVM_NETWORK is set to 'arbitrum-sepolia'"); Ok(Network::ArbitrumSepolia) + } else if use_arbitrum_sepolia_test { + info!("Using Arbitrum Sepolia Test EVM network as EVM_NETWORK is set to 'arbitrum-sepolia-test'"); + Ok(Network::ArbitrumSepoliaTest) } else if let Ok(evm_vars) = evm_vars { info!("Using custom EVM network from environment variables"); Ok(Network::Custom(CustomNetwork::new( diff --git a/evmlib/tests/network_token.rs b/evmlib/tests/network_token.rs index 77e2a1d723..878c4e950c 100644 --- a/evmlib/tests/network_token.rs +++ b/evmlib/tests/network_token.rs @@ -60,7 +60,7 @@ async fn test_balance_of() { assert_eq!( balance, - U256::from_str("20000000000000000000000000").unwrap() + U256::from_str("2500000000000000000000000").unwrap() ); } diff --git a/evmlib/tests/payment_vault.rs b/evmlib/tests/payment_vault.rs index 41c5881cbb..8dc3b9e6a1 100644 --- a/evmlib/tests/payment_vault.rs +++ b/evmlib/tests/payment_vault.rs @@ -11,7 +11,7 @@ use alloy::providers::fillers::{ use alloy::providers::{Identity, ProviderBuilder, ReqwestProvider, WalletProvider}; use alloy::signers::local::{LocalSigner, PrivateKeySigner}; use alloy::transports::http::{Client, Http}; -use evmlib::common::{Amount, U256}; +use evmlib::common::U256; use evmlib::contract::network_token::NetworkToken; use evmlib::contract::payment_vault::handler::PaymentVaultHandler; use evmlib::contract::payment_vault::{interface, MAX_TRANSFERS_PER_TRANSACTION}; @@ -116,26 +116,38 @@ async fn test_deploy() { } #[tokio::test] -async fn test_proxy_reachable() { - let network = Network::ArbitrumOne; +async fn test_get_quote_on_arb_sepolia() { + let network = Network::ArbitrumSepolia; let provider = http_provider(network.rpc_url().clone()); let payment_vault = PaymentVaultHandler::new(*network.data_payments_address(), provider); - let amount = payment_vault - .get_quote(vec![QuotingMetrics::default()]) - .await - .unwrap(); + let quoting_metrics = QuotingMetrics { + data_type: 1, // a GraphEntry record + data_size: 100, + close_records_stored: 10, + records_per_type: vec![(0, 5), (1, 5)], + max_records: 16 * 1024, + received_payment_count: 0, + live_time: 1400, + network_density: Some([ + 4, 4, 224, 228, 247, 252, 14, 44, 67, 21, 153, 47, 244, 18, 232, 1, 152, 195, 44, 43, + 29, 135, 19, 217, 240, 129, 64, 245, 240, 227, 129, 162, + ]), + network_size: Some(240), + }; + + let result = payment_vault.get_quote(vec![quoting_metrics]).await; - assert_eq!(amount, vec![Amount::from(1)]); + assert!(result.is_ok(), "Failed with error: {:?}", result.err()); } #[tokio::test] -async fn test_verify_payment() { +async fn test_pay_for_quotes_on_local() { let (_anvil, network_token, mut payment_vault) = setup().await; let mut quote_payments = vec![]; - for _ in 0..5 { + for _ in 0..MAX_TRANSFERS_PER_TRANSACTION { let quote_payment = random_quote_payment(); quote_payments.push(quote_payment); } @@ -149,36 +161,18 @@ async fn test_verify_payment() { // so we set it to the same as the network token contract payment_vault.set_provider(network_token.contract.provider().clone()); - let result = payment_vault.pay_for_quotes(quote_payments.clone()).await; + let result = payment_vault.pay_for_quotes(quote_payments).await; assert!(result.is_ok(), "Failed with error: {:?}", result.err()); - - let payment_verifications: Vec<_> = quote_payments - .into_iter() - .map(|v| interface::IPaymentVault::PaymentVerification { - metrics: QuotingMetrics::default().into(), - rewardsAddress: v.1, - quoteHash: v.0, - }) - .collect(); - - let results = payment_vault - .verify_payment(payment_verifications) - .await - .expect("Verify payment failed"); - - for result in results { - assert!(result.isValid); - } } #[tokio::test] -async fn test_pay_for_quotes() { +async fn test_verify_payment_on_local() { let (_anvil, network_token, mut payment_vault) = setup().await; let mut quote_payments = vec![]; - for _ in 0..MAX_TRANSFERS_PER_TRANSACTION { + for _ in 0..5 { let quote_payment = random_quote_payment(); quote_payments.push(quote_payment); } @@ -192,7 +186,36 @@ async fn test_pay_for_quotes() { // so we set it to the same as the network token contract payment_vault.set_provider(network_token.contract.provider().clone()); - let result = payment_vault.pay_for_quotes(quote_payments).await; + let result = payment_vault.pay_for_quotes(quote_payments.clone()).await; assert!(result.is_ok(), "Failed with error: {:?}", result.err()); + + let payment_verifications: Vec<_> = quote_payments + .into_iter() + .map(|v| interface::IPaymentVault::PaymentVerification { + metrics: QuotingMetrics { + data_size: 0, + data_type: 0, + close_records_stored: 0, + records_per_type: vec![], + max_records: 0, + received_payment_count: 0, + live_time: 0, + network_density: None, + network_size: None, + } + .into(), + rewardsAddress: v.1, + quoteHash: v.0, + }) + .collect(); + + let results = payment_vault + .verify_payment(payment_verifications) + .await + .expect("Verify payment failed"); + + for result in results { + assert!(result.isValid); + } } diff --git a/evmlib/tests/wallet.rs b/evmlib/tests/wallet.rs index e9e5f0a077..6713879279 100644 --- a/evmlib/tests/wallet.rs +++ b/evmlib/tests/wallet.rs @@ -90,7 +90,21 @@ async fn test_pay_for_quotes_and_data_payment_verification() { let result = verify_data_payment( &network, vec![*quote_hash], - vec![(*quote_hash, QuotingMetrics::default(), *reward_addr)], + vec![( + *quote_hash, + QuotingMetrics { + data_size: 0, + data_type: 0, + close_records_stored: 0, + records_per_type: vec![], + max_records: 0, + received_payment_count: 0, + live_time: 0, + network_density: None, + network_size: None, + }, + *reward_addr, + )], ) .await; diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000..a7533dc5d6 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,81 @@ +site_name: Autonomi Documentation +docs_dir: docs/online-documentation +theme: + name: material + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.indexes + - toc.integrate + - search.suggest + - search.highlight + - content.tabs.link + - content.code.annotation + - content.code.copy + language: en + palette: + - scheme: default + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + primary: teal + accent: purple + - scheme: slate + toggle: + icon: material/toggle-switch + name: Switch to light mode + primary: teal + accent: lime + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Core Concepts: + - Data Types: guides/data_types.md + - Client Modes: guides/client_modes.md + - Data Storage: guides/data_storage.md + - Developer Guides: + - Local Network Setup: guides/local_network.md + - Local Development: guides/local_development.md + - EVM Integration: guides/evm_integration.md + - Testing Guide: guides/testing_guide.md + - Payments: guides/payments.md + - API Reference: + - Overview: api/index.md + - Client API: + - Overview: api/autonomi-client/README.md + - Data Types: api/autonomi-client/data_types.md + - Error Handling: api/autonomi-client/errors.md + - Node API: + - Overview: api/ant-node/README.md + - Node Configuration: api/ant-node/configuration.md + - Network Operations: api/ant-node/network.md + - Cryptography: + - BLS Threshold Crypto: api/blsttc/README.md + - Self Encryption: api/self-encryption/README.md + +plugins: + - search + - git-revision-date-localized: + enable_creation_date: true + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - admonition + - pymdownx.arithmatex: + generic: true + - footnotes + - pymdownx.details + - pymdownx.superfences + - pymdownx.mark + - attr_list + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.tabbed: + alternate_style: true \ No newline at end of file diff --git a/nat-detection/Cargo.toml b/nat-detection/Cargo.toml index 2ad2fc2dcd..6a43a51352 100644 --- a/nat-detection/Cargo.toml +++ b/nat-detection/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "nat-detection" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" -version = "0.2.15" +version = "0.2.16" [[bin]] name = "nat-detection" @@ -17,14 +17,14 @@ path = "src/main.rs" nightly = [] [dependencies] -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-networking = { path = "../ant-networking", version = "0.3.4" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-networking = { path = "../ant-networking", version = "0.3.5" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } clap = { version = "4.5.4", features = ["derive"] } clap-verbosity-flag = "2.2.0" color-eyre = { version = "0.6", default-features = false } futures = "~0.3.13" -libp2p = { version = "0.54.1", features = [ +libp2p = { version = "0.55.0", features = [ "tokio", "tcp", "noise", diff --git a/node-launchpad/Cargo.toml b/node-launchpad/Cargo.toml index 8dc6e05afd..233ef6e2ea 100644 --- a/node-launchpad/Cargo.toml +++ b/node-launchpad/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "TUI for running nodes on the Autonomi network" name = "node-launchpad" -version = "0.5.3" +version = "0.5.4" edition = "2021" license = "GPL-3.0" homepage = "https://maidsafe.net" @@ -18,13 +18,13 @@ path = "src/bin/tui/main.rs" nightly = [] [dependencies] -ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.4" } -ant-build-info = { path = "../ant-build-info", version = "0.1.23" } -ant-evm = { path = "../ant-evm", version = "0.1.8" } -ant-node-manager = { version = "0.11.7", path = "../ant-node-manager" } -ant-protocol = { path = "../ant-protocol", version = "0.3.3" } +ant-bootstrap = { path = "../ant-bootstrap", version = "0.1.5" } +ant-build-info = { path = "../ant-build-info", version = "0.1.24" } +ant-evm = { path = "../ant-evm", version = "0.1.9" } +ant-node-manager = { version = "0.11.8", path = "../ant-node-manager" } +ant-protocol = { path = "../ant-protocol", version = "1.0.0" } ant-releases = { version = "0.4.0" } -ant-service-management = { version = "0.4.7", path = "../ant-service-management" } +ant-service-management = { version = "0.4.8", path = "../ant-service-management" } arboard = "3.4.1" atty = "0.2.14" better-panic = "0.3.0" @@ -36,8 +36,8 @@ clap = { version = "4.4.5", features = [ "unicode", "string", "unstable-styles", -]} -color-eyre = "0.6.2" +] } +color-eyre = "0.6.3" config = "0.14.0" crossterm = { version = "0.27.0", features = ["serde", "event-stream"] } derive_deref = "1.1.1" @@ -57,7 +57,7 @@ ratatui = { version = "0.29.0", features = ["serde", "macros", "unstable-widget- regex = "1.11.0" reqwest = { version = "0.12.2", default-features = false, features = [ "rustls-tls-manual-roots", -]} +] } serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.107" signal-hook = "0.3.17" diff --git a/node-launchpad/src/error.rs b/node-launchpad/src/error.rs index 14005a87a6..fb2a41c2fb 100644 --- a/node-launchpad/src/error.rs +++ b/node-launchpad/src/error.rs @@ -13,7 +13,7 @@ use ratatui::{ /// Error popup is a popup that is used to display error messages to the user. /// -/// It accepts a title, a message and a error message. +/// It accepts a title, a message and an error message. /// Handles key events to hide the popup (Enter and Esc keys). /// /// How to use: diff --git a/node-launchpad/src/node_mgmt.rs b/node-launchpad/src/node_mgmt.rs index 18780b4f2b..c5a281d483 100644 --- a/node-launchpad/src/node_mgmt.rs +++ b/node-launchpad/src/node_mgmt.rs @@ -419,7 +419,7 @@ async fn scale_down_nodes(config: &NodeConfig, count: u16) { config.data_dir_path.clone(), true, None, - Some(EvmNetwork::ArbitrumSepolia), + Some(EvmNetwork::default()), config.home_network, None, None, @@ -429,7 +429,6 @@ async fn scale_down_nodes(config: &NodeConfig, count: u16) { config.network_id, None, None, // We don't care about the port, as we are scaling down - config.owner.clone(), config.peers_args.clone(), RewardsAddress::from_str(config.rewards_address.as_str()).unwrap(), None, @@ -493,7 +492,7 @@ async fn add_nodes( config.data_dir_path.clone(), true, None, - Some(EvmNetwork::ArbitrumSepolia), + Some(EvmNetwork::default()), config.home_network, None, None, @@ -503,7 +502,6 @@ async fn add_nodes( config.network_id, None, port_range, - config.owner.clone(), config.peers_args.clone(), RewardsAddress::from_str(config.rewards_address.as_str()).unwrap(), None, diff --git a/nodejs/.gitignore b/nodejs/.gitignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/nodejs/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/nodejs/README.md b/nodejs/README.md new file mode 100644 index 0000000000..ec2269511b --- /dev/null +++ b/nodejs/README.md @@ -0,0 +1,116 @@ +# Autonomi Node.js Client + +TypeScript/JavaScript bindings for the Autonomi client. + +## Installation + +```bash +npm install @autonomi/client +``` + +## Usage + +```typescript +import { Client } from '@autonomi/client'; + +async function example() { + // Connect to the network + const client = await Client.connect({ + peers: ['/ip4/127.0.0.1/tcp/12000'] + }); + + // Create a payment option using a wallet + const payment = { + type: 'wallet' as const, + wallet: 'your_wallet_address' + }; + + // Upload public data + const data = Buffer.from('Hello, Safe Network!'); + const addr = await client.dataPutPublic(data, payment); + console.log(`Data uploaded to: ${addr}`); + + // Download public data + const retrieved = await client.dataGetPublic(addr); + console.log(`Retrieved: ${retrieved.toString()}`); +} +``` + +## Features + +- TypeScript support with full type definitions +- Async/await API +- Support for: + - Public and private data operations + - Graph + - Pointers + - Vaults + - User data management + +## API Reference + +### Client + +The main interface to interact with the Autonomi network. + +#### Connection + +```typescript +static connect(config: NetworkConfig): Promise +``` + +#### Data Operations + +```typescript +dataPutPublic(data: Buffer, payment: PaymentOption): Promise +dataGetPublic(address: string): Promise +``` + +#### Graph Operations + +```typescript +GraphEntryGet(address: string): Promise +GraphEntryPut(options: GraphEntryOptions, payment: PaymentOption): Promise +GraphEntryCost(key: string): Promise +``` + +#### Pointer Operations + +```typescript +pointerGet(address: string): Promise +pointerPut(options: PointerOptions, payment: PaymentOption): Promise +pointerCost(key: string): Promise +``` + +#### Vault Operations + +```typescript +vaultCost(key: string): Promise +writeBytesToVault(data: Buffer, payment: PaymentOption, options: VaultOptions): Promise +fetchAndDecryptVault(key: string): Promise<[Buffer, number]> +getUserDataFromVault(key: string): Promise +putUserDataToVault(key: string, payment: PaymentOption, userData: UserData): Promise +``` + +## Development + +```bash +# Install dependencies +npm install + +# Build +npm run build + +# Run tests +npm test + +# Run tests in watch mode +npm run test:watch + +# Lint +npm run lint +``` + +## License + +GPL-3.0 diff --git a/nodejs/jest.config.js b/nodejs/jest.config.js new file mode 100644 index 0000000000..d58f488638 --- /dev/null +++ b/nodejs/jest.config.js @@ -0,0 +1,11 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/src', '/tests'], + testMatch: ['**/*.test.ts'], + collectCoverage: true, + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov'], + coveragePathIgnorePatterns: ['/node_modules/', '/dist/'] +}; \ No newline at end of file diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json new file mode 100644 index 0000000000..9af942eede --- /dev/null +++ b/nodejs/package-lock.json @@ -0,0 +1,4012 @@ +{ + "name": "nodejs", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nodejs", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^22.10.2", + "jest": "^29.7.0", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "^5.7.2" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.3.tgz", + "integrity": "sha512-nHIxvKPniQXpmQLb0vhY3VaFb3S0YrTAwpOWJZh1wn3oJPjJk9Asva204PsBdmAE8vpzfHudT8DB0scYvy9q0g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", + "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.0", + "@babel/generator": "^7.26.0", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.0", + "@babel/parser": "^7.26.0", + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.26.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz", + "integrity": "sha512-6FF/urZvD0sTeO7k6/B15pMLC4CHUv1426lzr3N01aHJTl046uCAh9LXW/fzeXXjPNCJ6iABW5XaWOsIZB93aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.3", + "@babel/types": "^7.26.3", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", + "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", + "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", + "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", + "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.3" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", + "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", + "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", + "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.26.4", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.4.tgz", + "integrity": "sha512-fH+b7Y4p3yqvApJALCPJcwb0/XaOSgtK4pzV6WVjPR5GLFQBRI7pfoX2V2iM48NXvX07NUxxm1Vw98YjqTcU5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.3", + "@babel/parser": "^7.26.3", + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.3", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", + "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", + "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.10.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", + "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.3.tgz", + "integrity": "sha512-1CPmv8iobE2fyRMV97dAcMVegvvWKxmq94hkLiAkUGwKVTyDLw33K+ZxiFrREKmmps4rIw6grcCFCnTMSZ/YiA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bs-logger": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", + "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-json-stable-stringify": "2.x" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001690", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001690.tgz", + "integrity": "sha512-5ExiE3qQN6oF8Clf8ifIDcMRCRE/dMGcETG/XGMD8/XiXm6HXQgQTh1yZYLXXpSOsEUlJm1Xr7kGULZTuGtP/w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz", + "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", + "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dedent": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", + "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.76", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.76.tgz", + "integrity": "sha512-CjVQyG7n7Sr+eBXE86HIulnL5N8xZY1sgmOPGuq/F0Rr0FJq63lg0kEtOIDfZBk44FnDLf6FUJ+dsJcuiUDdDQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jake": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", + "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-jest": { + "version": "29.2.5", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz", + "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bs-logger": "^0.2.6", + "ejs": "^3.1.10", + "fast-json-stable-stringify": "^2.1.0", + "jest-util": "^29.0.0", + "json5": "^2.2.3", + "lodash.memoize": "^4.1.2", + "make-error": "^1.3.6", + "semver": "^7.6.3", + "yargs-parser": "^21.1.1" + }, + "bin": { + "ts-jest": "cli.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": ">=7.0.0-beta.0 <8", + "@jest/transform": "^29.0.0", + "@jest/types": "^29.0.0", + "babel-jest": "^29.0.0", + "jest": "^29.0.0", + "typescript": ">=4.3 <6" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "@jest/transform": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "babel-jest": { + "optional": true + }, + "esbuild": { + "optional": true + } + } + }, + "node_modules/ts-jest/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", + "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", + "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.0" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/nodejs/package.json b/nodejs/package.json new file mode 100644 index 0000000000..18141ca7f1 --- /dev/null +++ b/nodejs/package.json @@ -0,0 +1,38 @@ +{ + "name": "@autonomi/client", + "version": "0.1.0", + "description": "Node.js bindings for Autonomi client", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "test": "jest", + "test:watch": "jest --watch", + "lint": "eslint src/**/*.ts", + "clean": "rm -rf dist", + "prepare": "npm run build" + }, + "keywords": [ + "autonomi", + "client", + "network", + "graph", + "pointer", + "vault" + ], + "author": "Safe Network", + "license": "GPL-3.0", + "devDependencies": { + "@types/jest": "^29.5.11", + "@types/node": "^20.10.6", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ] +} \ No newline at end of file diff --git a/nodejs/src/client.ts b/nodejs/src/client.ts new file mode 100644 index 0000000000..de5f385ba9 --- /dev/null +++ b/nodejs/src/client.ts @@ -0,0 +1,91 @@ +import { NetworkConfig, PaymentOption, GraphEntryOptions, PointerOptions, VaultOptions, UserData } from './types'; + +export class Client { + private nativeClient: any; // Will be replaced with actual native binding type + + private constructor(nativeClient: any) { + this.nativeClient = nativeClient; + } + + static async connect(config: NetworkConfig): Promise { + // TODO: Initialize native client + throw new Error('Not implemented'); + } + + // Data Operations + async dataPutPublic(data: Buffer, payment: PaymentOption): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async dataGetPublic(address: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + // Graph Operations + async graphEntryGet(address: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async graphEntryPut(options: GraphEntryOptions, payment: PaymentOption): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async graphEntryCost(key: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + // Pointer Operations + async pointerGet(address: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async pointerPut(options: PointerOptions, payment: PaymentOption): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async pointerCost(key: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + // Vault Operations + async vaultCost(key: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async writeBytesToVault( + data: Buffer, + payment: PaymentOption, + options: VaultOptions + ): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async fetchAndDecryptVault(key: string): Promise<[Buffer, number]> { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async getUserDataFromVault(key: string): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } + + async putUserDataToVault( + key: string, + payment: PaymentOption, + userData: UserData + ): Promise { + // TODO: Implement native binding call + throw new Error('Not implemented'); + } +} \ No newline at end of file diff --git a/nodejs/src/index.ts b/nodejs/src/index.ts new file mode 100644 index 0000000000..460dc50882 --- /dev/null +++ b/nodejs/src/index.ts @@ -0,0 +1,6 @@ +export * from './client'; +export * from './types'; +export * from './wallet'; +export * from './GraphEntry'; +export * from './pointer'; +export * from './vault'; \ No newline at end of file diff --git a/nodejs/src/types.ts b/nodejs/src/types.ts new file mode 100644 index 0000000000..bc3718f036 --- /dev/null +++ b/nodejs/src/types.ts @@ -0,0 +1,38 @@ +export type Address = string; +export type HexString = string; +export type SecretKey = string; +export type PublicKey = string; + +export interface PaymentOption { + type: 'wallet'; + wallet: string; +} + +export interface GraphEntryOptions { + owner: PublicKey; + counter: number; + target: string; + key: SecretKey; +} + +export interface PointerOptions { + owner: PublicKey; + counter: number; + target: string; + key: SecretKey; +} + +export interface VaultOptions { + key: SecretKey; + contentType?: number; +} + +export interface UserData { + fileArchives: Array<[string, string]>; + privateFileArchives: Array<[string, string]>; +} + +export interface NetworkConfig { + peers: string[]; + network?: 'arbitrum' | 'arbitrum_testnet'; +} \ No newline at end of file diff --git a/nodejs/tests/client.test.ts b/nodejs/tests/client.test.ts new file mode 100644 index 0000000000..d1d4cd699f --- /dev/null +++ b/nodejs/tests/client.test.ts @@ -0,0 +1,33 @@ +import { Client } from '../src/client'; + +describe('Client', () => { + describe('connect', () => { + it('should throw not implemented error', async () => { + await expect(Client.connect({ peers: [] })).rejects.toThrow('Not implemented'); + }); + }); + + describe('GraphEntryOperations', () => { + it('should throw not implemented error for GraphEntryGet', async () => { + const client = await Client.connect({ peers: [] }).catch(() => null); + if (!client) return; + await expect(client.GraphEntryGet('address')).rejects.toThrow('Not implemented'); + }); + + it('should throw not implemented error for GraphEntryPut', async () => { + const client = await Client.connect({ peers: [] }).catch(() => null); + if (!client) return; + await expect( + client.GraphEntryPut( + { + owner: 'owner', + counter: 0, + target: 'target', + key: 'key' + }, + { type: 'wallet', wallet: 'wallet' } + ) + ).rejects.toThrow('Not implemented'); + }); + }); +}); \ No newline at end of file diff --git a/nodejs/tsconfig.json b/nodejs/tsconfig.json new file mode 100644 index 0000000000..6094d01583 --- /dev/null +++ b/nodejs/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": [ + "ES2020" + ], + "declaration": true, + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "node" + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "tests" + ] +} \ No newline at end of file diff --git a/release-cycle-info b/release-cycle-info index 54e44b5314..9619250bcb 100644 --- a/release-cycle-info +++ b/release-cycle-info @@ -12,7 +12,7 @@ # # Both of these numbers are used in the packaged version number, which is a collective version # number for all the released binaries. -release-year: 2024 -release-month: 12 -release-cycle: 1 -release-cycle-counter: 11 +release-year: 2025 +release-month: 1 +release-cycle: 2 +release-cycle-counter: 3 diff --git a/resources/scripts/list-numbered-prs.py b/resources/scripts/list-numbered-prs.py index 26cfc2c2a8..dfabf86c0c 100755 --- a/resources/scripts/list-numbered-prs.py +++ b/resources/scripts/list-numbered-prs.py @@ -49,7 +49,7 @@ def main(pr_numbers): pr_number = pr["number"] closed_date = pr["closed_at"].date() breaking_text = "[BREAKING]" if pr["breaking"] else "" - print(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/safe_network/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") + print(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/autonomi/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") print() grouped_pulls = defaultdict(list) @@ -83,7 +83,7 @@ def main(pr_numbers): pr_number = pr["number"] closed_date = pr["closed_at"].date() breaking_text = "[BREAKING]" if pr["breaking"] else "" - print(f" {closed_date} [#{pr_number}](https://github.com/maidsafe/safe_network/pull/{pr_number}) -- {pr['title']} {breaking_text}") + print(f" {closed_date} [#{pr_number}](https://github.com/maidsafe/autonomi/pull/{pr_number}) -- {pr['title']} {breaking_text}") print() def read_pr_numbers(file_path): diff --git a/resources/scripts/list-safe-network-closed-prs.py b/resources/scripts/list-safe-network-closed-prs.py index 6355703c43..90c1007989 100755 --- a/resources/scripts/list-safe-network-closed-prs.py +++ b/resources/scripts/list-safe-network-closed-prs.py @@ -19,7 +19,7 @@ def main(last_released_pr_number): raise Exception("The GITHUB_PAT_SAFE_NETWORK_PR_LIST environment variable must be set") g = Github(token) - repo = g.get_repo("maidsafe/safe_network") + repo = g.get_repo("maidsafe/autonomi") last_released_pr = repo.get_pull(last_released_pr_number) if not last_released_pr: @@ -64,7 +64,7 @@ def main(last_released_pr_number): pr_number = pr["number"] closed_date = pr["closed_at"].date() breaking_text = "[BREAKING]" if pr["breaking"] else "" - print(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/safe_network/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") + print(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/autonomi/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") print() grouped_pulls = defaultdict(list) @@ -98,7 +98,7 @@ def main(last_released_pr_number): pr_number = pr["number"] closed_date = pr["closed_at"].date() breaking_text = "[BREAKING]" if pr["breaking"] else "" - print(f" {closed_date} [#{pr_number}](https://github.com/maidsafe/safe_network/pull/{pr_number}) -- {pr['title']} {breaking_text}") + print(f" {closed_date} [#{pr_number}](https://github.com/maidsafe/autonomi/pull/{pr_number}) -- {pr['title']} {breaking_text}") print() diff --git a/resources/scripts/release-candidate-description.py b/resources/scripts/release-candidate-description.py index 10a91e0b96..1361c14bf5 100755 --- a/resources/scripts/release-candidate-description.py +++ b/resources/scripts/release-candidate-description.py @@ -64,7 +64,7 @@ def get_pr_list(pr_numbers): pr_number = pr["number"] closed_date = pr["closed_at"].date() breaking_text = "[BREAKING]" if pr["breaking"] else "" - markdown_lines.append(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/safe_network/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") + markdown_lines.append(f"{closed_date} [#{pr_number}](https://github.com/maidsafe/autonomi/pull/{pr_number}) -- {pr['title']} [@{pr['author']}] {breaking_text}") return markdown_lines diff --git a/resources/scripts/upload-random-data.sh b/resources/scripts/upload-random-data.sh index dbcf5b06be..d58d926e23 100755 --- a/resources/scripts/upload-random-data.sh +++ b/resources/scripts/upload-random-data.sh @@ -17,8 +17,8 @@ fi check_and_install_safe() { if ! command -v safe &> /dev/null; then echo "'safe' command not found. Installing..." - curl -sSL https://raw.githubusercontent.com/maidsafe/safeup/main/install.sh | sudo bash - safeup client + curl -sSL https://raw.githubusercontent.com/maidsafe/antup/main/install.sh | sudo bash + antup client else echo "'safe' command is already installed." fi diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 7e6538d29f..98a75121b0 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -6,15 +6,15 @@ homepage = "https://maidsafe.net" license = "GPL-3.0" name = "test-utils" readme = "README.md" -repository = "https://github.com/maidsafe/safe_network" -version = "0.4.15" +repository = "https://github.com/maidsafe/autonomi" +version = "0.4.16" [dependencies] bytes = { version = "1.0.1", features = ["serde"] } -color-eyre = "~0.6.2" +color-eyre = "0.6.3" dirs-next = "~2.0.0" -evmlib = { path = "../evmlib", version = "0.1.8" } -libp2p = { version = "0.54.1", features = ["identify", "kad"] } +evmlib = { path = "../evmlib", version = "0.1.9" } +libp2p = { version = "0.55.0", features = ["identify", "kad"] } rand = "0.8.5" serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0" diff --git a/test-utils/src/evm.rs b/test-utils/src/evm.rs index 05eb710bde..e3db3dd2ba 100644 --- a/test-utils/src/evm.rs +++ b/test-utils/src/evm.rs @@ -6,16 +6,12 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use color_eyre::{ - eyre::{bail, Context}, - Result, -}; -use evmlib::{utils::get_evm_network_from_env, wallet::Wallet, Network}; +use color_eyre::{eyre::bail, Result}; +use evmlib::{utils::get_evm_network, wallet::Wallet, Network}; use std::env; pub fn get_funded_wallet() -> evmlib::wallet::Wallet { - let network = - get_evm_network_from_env().expect("Failed to get EVM network from environment variables"); + let network = get_evm_network(true).expect("Failed to get local EVM network from CSV"); if matches!(network, Network::ArbitrumOne) { panic!("You're trying to use ArbitrumOne network. Use a custom network for testing."); } @@ -29,8 +25,7 @@ pub fn get_funded_wallet() -> evmlib::wallet::Wallet { } pub fn get_new_wallet() -> Result { - let network = get_evm_network_from_env() - .wrap_err("Failed to get EVM network from environment variables")?; + let network = get_evm_network(true).expect("Failed to get local EVM network from CSV"); if matches!(network, Network::ArbitrumOne) { bail!("You're trying to use ArbitrumOne network. Use a custom network for testing."); }