Merge remote-tracking branch 'origin/master' into bump-candid

CHANGELOG.md

@@ -2,10 +2,20 @@
 
 # UNRELEASED
 
+### feat: dfx deps: wasm_hash_url and loose the hash check
+
+Providers can provide the hash through `wasm_hash_url` instead of hard coding the hash directly.
+
+If the hash of downloaded wasm doesn’t match the provided hash (`wasm_hash`, `wasm_hash_url` or read from mainnet state tree), dfx deps won’t abort. Instead, it will print a warning message.
+
 ### feat!: update `dfx cycles` commands with mainnet `cycles-ledger` canister ID
 
 The `dfx cycles` command no longer needs nor accepts the `--cycles-ledger-canister-id <canister id>` parameter.
 
+### chore: removed the dfx start --emulator mode
+
+This was deprecated in dfx 0.15.1.
+
 ### chore: removed ic-ref from the binary cache
 
 ### chore: updated dependencies for new rust projects

docs/cli-reference/dfx-start.md

@@ -18,7 +18,6 @@ You can use the following optional flags with the `dfx start` command.
 |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `--background`           | Starts the local canister execution environment and web server processes in the background and waits for a reply before returning to the shell.                                                                                              |
 | `--clean`                | Starts the local canister execution environment and web server processes in a clean state by removing checkpoints from your project cache. You can use this flag to set your project cache to a new state when troubleshooting or debugging. |
-| `--emulator`      | Starts the [IC reference emulator](https://github.com/dfinity/ic-hs) rather than the replica. (deprecated: will be discontinued soon)                                                                                                        |
 | `--enable-bitcoin` | Enables bitcoin integration.                                                                                                                                                                                                                 |
 | `--enable-canister-http` | Enables canister HTTP requests. (deprecated: now enabled by default)                                                                                                                                                                         |
 | `--use-old-metering` | Enables the old metering in the local canister execution environment. Please see the forum thread for more details or to report any issues: [forum.dfinity.org/t/new-wasm-instrumentation/](https://forum.dfinity.org/t/new-wasm-instrumentation/22080) |

docs/concepts/pull-dependencies.md

@@ -53,6 +53,14 @@ In most cases, the wasm module at `wasm_url` will be the same as the on-chain wa
 
 In other cases, the wasm module at `wasm_url` is not the same as the on-chain wasm module. For example, the Internet Identity canister provides Development flavor to be integrated locally. In these cases, `wasm_hash` provides the expected hash, and dfx verifies the downloaded wasm against this.
 
+### `wasm_hash_url` 
+
+A URL to get the SHA256 hash of the wasm module located at `wasm_url`.
+
+This field is optional.
+
+Aside from specifying SHA256 hash of the wasm module directly using `wasm_hash`, providers can also specify the hash with this URL. If both are defined, the `wasm_hash_url` field will be ignored.
+
 ### `dependencies`
 
 An array of Canister IDs (`Principal`) of direct dependencies.

docs/dfx-json-schema.json

@@ -982,7 +982,15 @@
         },
         "wasm_hash": {
           "title": "wasm_hash",
-          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.",
+          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified via a URL using the `wasm_hash_url` field. If both are defined, the `wasm_hash_url` field will be ignored.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "wasm_hash_url": {
+          "title": "wasm_hash_url",
+          "description": "Specify the SHA256 hash of the wasm module via this URL. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified directly using the `wasm_hash` field. If both are defined, the `wasm_hash_url` field will be ignored.",
           "type": [
             "string",
             "null"

e2e/tests-dfx/cycles-ledger.bash

@@ -444,6 +444,7 @@ current_time_nanoseconds() {
 
   # using dfx canister create
   dfx identity use alice
+  # shellcheck disable=SC2030
   export DFX_DISABLE_AUTO_WALLET=1
   t=$(current_time_nanoseconds)
   assert_command dfx canister create e2e_project_backend --with-cycles 1T --created-at-time "$t"
@@ -501,3 +502,46 @@ current_time_nanoseconds() {
   assert_command dfx cycles balance --precise
   assert_eq "9399600000000 cycles."
 }
+
+@test "canister deletion" {
+  skip "can't be properly tested with feature flag turned off (CYCLES_LEDGER_ENABLED). TODO(SDK-1331): re-enable this test"
+  dfx_new temporary
+  add_cycles_ledger_canisters_to_project
+  install_cycles_ledger_canisters
+
+  ALICE=$(dfx identity get-principal --identity alice)
+
+  assert_command deploy_cycles_ledger
+  CYCLES_LEDGER_ID=$(dfx canister id cycles-ledger)
+  echo "Cycles ledger deployed at id $CYCLES_LEDGER_ID"
+  assert_command dfx deploy cycles-depositor --argument "(record {ledger_id = principal \"$(dfx canister id cycles-ledger)\"})"
+  echo "Cycles depositor deployed at id $(dfx canister id cycles-depositor)"
+  assert_command dfx ledger fabricate-cycles --canister cycles-depositor --t 9999
+  assert_command dfx deploy
+  assert_command dfx canister call cycles-depositor deposit "(record {to = record{owner = principal \"$ALICE\";};cycles = 22_400_000_000_000;})" --identity cycle-giver
+
+  cd ..
+  dfx_new
+  # setup done
+
+  dfx identity use alice
+  # shellcheck disable=SC2031
+  export DFX_DISABLE_AUTO_WALLET=1
+  assert_command dfx canister create --all --with-cycles 10T
+  assert_command dfx cycles balance --precise
+  assert_eq "2399800000000 cycles."
+
+  # delete by name
+  assert_command dfx canister stop --all
+  assert_command dfx canister delete e2e_project_backend
+  assert_command dfx cycles balance
+  assert_eq "12.389 TC (trillion cycles)."
+
+  # delete by id
+  FRONTEND_ID=$(dfx canister id e2e_project_frontend)
+  rm .dfx/local/canister_ids.json
+  assert_command dfx canister stop "${FRONTEND_ID}"
+  assert_command dfx canister delete "${FRONTEND_ID}"
+  assert_command dfx cycles balance
+  assert_eq "22.379 TC (trillion cycles)."
+}

e2e/tests-dfx/deps.bash

@@ -127,7 +127,7 @@ Failed to download from url: http://example.com/c.wasm."
 
   cd ../onchain
   dfx canister stop a
-  dfx canister delete a
+  dfx canister delete a --no-withdrawal
 
   cd ../app
   assert_command_fail dfx deps pull --network local
@@ -150,8 +150,6 @@ Failed to download from url: http://example.com/c.wasm."
 
   setup_onchain
 
-  # TODO: test gzipped wasm can be pulled when we have "gzip" option in dfx.json (SDK-1102)
-
   # pull canisters in app project
   cd app
   assert_file_not_exists "deps/pulled.json"
@@ -180,17 +178,16 @@ Failed to download from url: http://example.com/c.wasm."
   assert_command dfx deps pull --network local -vvv
   assert_contains "The canister wasm was found in the cache." # cache hit
 
-  # sad path 1: wasm hash doesn't match on chain
+  # warning: hash mismatch
   rm -r "${PULLED_DIR:?}/"
   cd ../onchain
   cp .dfx/local/canisters/c/c.wasm ../www/a.wasm
 
   cd ../app
-  assert_command_fail dfx deps pull --network local
-  assert_contains "Failed to pull canister $CANISTER_ID_A."
-  assert_contains "Hash mismatch."
+  assert_command dfx deps pull --network local
+  assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download."
 
-  # sad path 2: url server doesn't have the file
+  # sad path: url server doesn't have the file
   rm -r "${PULLED_DIR:?}/"
   rm ../www/a.wasm
 
@@ -199,8 +196,7 @@ Failed to download from url: http://example.com/c.wasm."
   assert_contains "Failed to download from url:"
 }
 
-
-@test "dfx deps pull can check hash when dfx:wasm_hash specified" {
+@test "dfx deps pull works when wasm_hash or wasm_hash_url specified" {
   use_test_specific_cache_root # dfx deps pull will download files to cache
 
   # start a "mainnet" replica which host the onchain canisters
@@ -228,11 +224,20 @@ Failed to download from url: http://example.com/c.wasm."
   cp .dfx/local/canisters/b/b.wasm.gz ../www/b.wasm.gz
   cp .dfx/local/canisters/c/c.wasm ../www/c.wasm
 
-  CUSTOM_HASH="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)"
-  jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH"'"' dfx.json | sponge dfx.json
-  dfx build a # .dfx/local/canisters/a/a.wasm is replaced. The new wasm has wasm_hash defined and will be installed.
+  # A: set dfx:wasm_hash
+  CUSTOM_HASH_A="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)"
+  jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH_A"'"' dfx.json | sponge dfx.json
+  # B: set dfx:wasm_hash_url
+  echo -n "$(sha256sum .dfx/local/canisters/b/b.wasm.gz | cut -d " " -f 1)" > ../www/b.wasm.gz.sha256
+  jq '.canisters.b.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/b.wasm.gz.sha256"'"' dfx.json | sponge dfx.json
+  # C: set both dfx:wasm_hash and dfx:wasm_hash_url. This should be avoided by providers.
+  CUSTOM_HASH_C="$(sha256sum .dfx/local/canisters/c/c.wasm | cut -d " " -f 1)"
+  jq '.canisters.c.pullable.wasm_hash="'"$CUSTOM_HASH_C"'"' dfx.json | sponge dfx.json
+  echo -n "$CUSTOM_HASH_C" > ../www/c.wasm.sha256
+  jq '.canisters.c.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/c.wasm.sha256"'"' dfx.json | sponge dfx.json
+
+  dfx build
 
-  # cd ../../../
   dfx canister install a --argument 1
   dfx canister install b
   dfx canister install c --argument 3
@@ -243,18 +248,20 @@ Failed to download from url: http://example.com/c.wasm."
 
   assert_command dfx deps pull --network local -vvv
   assert_contains "Canister $CANISTER_ID_A specified a custom hash:"
+  assert_contains "Canister $CANISTER_ID_B specified a custom hash via url:"
+  assert_contains "WARN: Canister $CANISTER_ID_C specified both \`wasm_hash\` and \`wasm_hash_url\`. \`wasm_hash\` will be used."
+  assert_contains "Canister $CANISTER_ID_C specified a custom hash:"
 
-  # error case: hash mismatch
+  # warning: hash mismatch
   PULLED_DIR="$DFX_CACHE_ROOT/.cache/dfinity/pulled/"
   rm -r "${PULLED_DIR:?}/"
   cd ../onchain
   cp .dfx/local/canisters/a/a.wasm ../www/a.wasm # now the webserver has the onchain version of canister_a which won't match wasm_hash
 
   cd ../app
-  assert_command_fail dfx deps pull --network local -vvv
+  assert_command dfx deps pull --network local -vvv
   assert_contains "Canister $CANISTER_ID_A specified a custom hash:"
-  assert_contains "Failed to pull canister $CANISTER_ID_A."
-  assert_contains "Hash mismatch."
+  assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download."
 }
 
 @test "dfx deps init works" {
@@ -325,11 +332,11 @@ candid:args => (nat)"
   # delete onchain canisters so that the replica has no canisters as a clean local replica
   cd ../onchain
   dfx canister stop a
-  dfx canister delete a
+  dfx canister delete a --no-withdrawal
   dfx canister stop b
-  dfx canister delete b
+  dfx canister delete b --no-withdrawal
   dfx canister stop c
-  dfx canister delete c
+  dfx canister delete c --no-withdrawal
 
   cd ../app
   assert_command dfx deps init # b is set here
@@ -355,10 +362,10 @@ Installing canister: $CANISTER_ID_C (dep_c)"
 
   # deployed pull dependencies can be stopped and deleted
   assert_command dfx canister stop dep_b --identity anonymous
-  assert_command dfx canister delete dep_b --identity anonymous
+  assert_command dfx canister delete dep_b --identity anonymous --no-withdrawal
 
   assert_command dfx canister stop $CANISTER_ID_A --identity anonymous
-  assert_command dfx canister delete $CANISTER_ID_A --identity anonymous
+  assert_command dfx canister delete $CANISTER_ID_A --identity anonymous --no-withdrawal
 
   # error cases
   ## set wrong init argument
@@ -397,11 +404,11 @@ Installing canister: $CANISTER_ID_C (dep_c)"
   # delete onchain canisters so that the replica has no canisters as a clean local replica
   cd ../onchain
   dfx canister stop a
-  dfx canister delete a
+  dfx canister delete a --no-withdrawal
   dfx canister stop b
-  dfx canister delete b
+  dfx canister delete b --no-withdrawal
   dfx canister stop c
-  dfx canister delete c
+  dfx canister delete c --no-withdrawal
 
   cd ../app
   assert_command_fail dfx canister create dep_b
@@ -434,7 +441,7 @@ Installing canister: $CANISTER_ID_C (dep_c)"
 
   # start a clean local replica
   dfx canister stop app
-  dfx canister delete app
+  dfx canister delete app --no-withdrawal
   assert_command dfx deploy # only deploy app canister
 }
 
@@ -443,8 +450,8 @@ Installing canister: $CANISTER_ID_C (dep_c)"
 
   # verify the help message
   assert_command dfx deps pull -h
-  assert_contains "Pull canisters upon which the project depends. This command connects to the \"ic\" mainnet by default.
-You can still choose other network by setting \`--network\`"
+  assert_contains "Pull canisters upon which the project depends. This command connects to the \"ic\" mainnet by default."
+  assert_contains "You can still choose other network by setting \`--network\`"
 
   assert_command dfx deps pull
   assert_contains "There are no pull dependencies defined in dfx.json"

src/dfx-core/src/config/model/dfinity.rs

@@ -169,7 +169,15 @@ pub struct Pullable {
     /// # wasm_hash
     /// SHA256 hash of the wasm module located at wasm_url.
     /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.
+    /// The hash can also be specified via a URL using the `wasm_hash_url` field.
+    /// If both are defined, the `wasm_hash_url` field will be ignored.
     pub wasm_hash: Option<String>,
+    /// # wasm_hash_url
+    /// Specify the SHA256 hash of the wasm module via this URL.
+    /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.
+    /// The hash can also be specified directly using the `wasm_hash` field.
+    /// If both are defined, the `wasm_hash_url` field will be ignored.
+    pub wasm_hash_url: Option<String>,
     /// # dependencies
     /// Canister IDs (Principal) of direct dependencies.
     #[schemars(with = "Vec::<String>")]

src/dfx-core/src/config/model/local_server_descriptor.rs

@@ -98,11 +98,6 @@ impl LocalServerDescriptor {
         self.data_directory.join("icx-proxy-pid")
     }
 
-    /// This file contains the listening port of the ic-ref process
-    pub fn ic_ref_port_path(&self) -> PathBuf {
-        self.data_directory.join("ic-ref.port")
-    }
-
     /// This file contains the pid of the ic-btc-adapter process
     pub fn btc_adapter_pid_path(&self) -> PathBuf {
         self.data_directory.join("ic-btc-adapter-pid")
@@ -297,12 +292,11 @@ impl LocalServerDescriptor {
     /// Gets the port of a local replica.
     ///
     /// # Prerequisites
-    /// - A local replica or emulator needs to be running, e.g. with `dfx start`.
+    /// - A local replica needs to be running, e.g. with `dfx start`.
     pub fn get_running_replica_port(
         &self,
         logger: Option<&Logger>,
     ) -> Result<Option<u16>, NetworkConfigError> {
-        let emulator_port_path = self.ic_ref_port_path();
         let replica_port_path = self.replica_port_path();
 
         match read_port_from(&replica_port_path)? {
@@ -312,15 +306,7 @@ impl LocalServerDescriptor {
                 }
                 Ok(Some(port))
             }
-            None => match read_port_from(&emulator_port_path)? {
-                Some(port) => {
-                    if let Some(logger) = logger {
-                        info!(logger, "Found local emulator running on port {}", port);
-                    }
-                    Ok(Some(port))
-                }
-                None => Ok(self.replica.port),
-            },
+            None => Ok(self.replica.port),
         }
     }
 }