diff --git a/CHANGELOG.md b/CHANGELOG.md index a6b2f662d6..e822d103d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,20 @@ # UNRELEASED +### feat: dfx deps: wasm_hash_url and loose the hash check + +Providers can provide the hash through `wasm_hash_url` instead of hard coding the hash directly. + +If the hash of downloaded wasm doesn’t match the provided hash (`wasm_hash`, `wasm_hash_url` or read from mainnet state tree), dfx deps won’t abort. Instead, it will print a warning message. + ### feat!: update `dfx cycles` commands with mainnet `cycles-ledger` canister ID The `dfx cycles` command no longer needs nor accepts the `--cycles-ledger-canister-id ` parameter. +### chore: removed the dfx start --emulator mode + +This was deprecated in dfx 0.15.1. + ### chore: removed ic-ref from the binary cache ### chore: updated dependencies for new rust projects diff --git a/docs/cli-reference/dfx-start.md b/docs/cli-reference/dfx-start.md index 3621c67bc2..2ec382bf9d 100644 --- a/docs/cli-reference/dfx-start.md +++ b/docs/cli-reference/dfx-start.md @@ -18,7 +18,6 @@ You can use the following optional flags with the `dfx start` command. |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `--background` | Starts the local canister execution environment and web server processes in the background and waits for a reply before returning to the shell. | | `--clean` | Starts the local canister execution environment and web server processes in a clean state by removing checkpoints from your project cache. You can use this flag to set your project cache to a new state when troubleshooting or debugging. | -| `--emulator` | Starts the [IC reference emulator](https://github.com/dfinity/ic-hs) rather than the replica. (deprecated: will be discontinued soon) | | `--enable-bitcoin` | Enables bitcoin integration. | | `--enable-canister-http` | Enables canister HTTP requests. (deprecated: now enabled by default) | | `--use-old-metering` | Enables the old metering in the local canister execution environment. Please see the forum thread for more details or to report any issues: [forum.dfinity.org/t/new-wasm-instrumentation/](https://forum.dfinity.org/t/new-wasm-instrumentation/22080) | diff --git a/docs/concepts/pull-dependencies.md b/docs/concepts/pull-dependencies.md index d48dc0699c..70a077c34f 100644 --- a/docs/concepts/pull-dependencies.md +++ b/docs/concepts/pull-dependencies.md @@ -53,6 +53,14 @@ In most cases, the wasm module at `wasm_url` will be the same as the on-chain wa In other cases, the wasm module at `wasm_url` is not the same as the on-chain wasm module. For example, the Internet Identity canister provides Development flavor to be integrated locally. In these cases, `wasm_hash` provides the expected hash, and dfx verifies the downloaded wasm against this. +### `wasm_hash_url` + +A URL to get the SHA256 hash of the wasm module located at `wasm_url`. + +This field is optional. + +Aside from specifying SHA256 hash of the wasm module directly using `wasm_hash`, providers can also specify the hash with this URL. If both are defined, the `wasm_hash_url` field will be ignored. + ### `dependencies` An array of Canister IDs (`Principal`) of direct dependencies. diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index 40f4e890ac..728f48d0dc 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -982,7 +982,15 @@ }, "wasm_hash": { "title": "wasm_hash", - "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.", + "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified via a URL using the `wasm_hash_url` field. If both are defined, the `wasm_hash_url` field will be ignored.", + "type": [ + "string", + "null" + ] + }, + "wasm_hash_url": { + "title": "wasm_hash_url", + "description": "Specify the SHA256 hash of the wasm module via this URL. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified directly using the `wasm_hash` field. If both are defined, the `wasm_hash_url` field will be ignored.", "type": [ "string", "null" diff --git a/e2e/tests-dfx/cycles-ledger.bash b/e2e/tests-dfx/cycles-ledger.bash index f015872900..0d12dd70a0 100644 --- a/e2e/tests-dfx/cycles-ledger.bash +++ b/e2e/tests-dfx/cycles-ledger.bash @@ -444,6 +444,7 @@ current_time_nanoseconds() { # using dfx canister create dfx identity use alice + # shellcheck disable=SC2030 export DFX_DISABLE_AUTO_WALLET=1 t=$(current_time_nanoseconds) assert_command dfx canister create e2e_project_backend --with-cycles 1T --created-at-time "$t" @@ -501,3 +502,46 @@ current_time_nanoseconds() { assert_command dfx cycles balance --precise assert_eq "9399600000000 cycles." } + +@test "canister deletion" { + skip "can't be properly tested with feature flag turned off (CYCLES_LEDGER_ENABLED). TODO(SDK-1331): re-enable this test" + dfx_new temporary + add_cycles_ledger_canisters_to_project + install_cycles_ledger_canisters + + ALICE=$(dfx identity get-principal --identity alice) + + assert_command deploy_cycles_ledger + CYCLES_LEDGER_ID=$(dfx canister id cycles-ledger) + echo "Cycles ledger deployed at id $CYCLES_LEDGER_ID" + assert_command dfx deploy cycles-depositor --argument "(record {ledger_id = principal \"$(dfx canister id cycles-ledger)\"})" + echo "Cycles depositor deployed at id $(dfx canister id cycles-depositor)" + assert_command dfx ledger fabricate-cycles --canister cycles-depositor --t 9999 + assert_command dfx deploy + assert_command dfx canister call cycles-depositor deposit "(record {to = record{owner = principal \"$ALICE\";};cycles = 22_400_000_000_000;})" --identity cycle-giver + + cd .. + dfx_new + # setup done + + dfx identity use alice + # shellcheck disable=SC2031 + export DFX_DISABLE_AUTO_WALLET=1 + assert_command dfx canister create --all --with-cycles 10T + assert_command dfx cycles balance --precise + assert_eq "2399800000000 cycles." + + # delete by name + assert_command dfx canister stop --all + assert_command dfx canister delete e2e_project_backend + assert_command dfx cycles balance + assert_eq "12.389 TC (trillion cycles)." + + # delete by id + FRONTEND_ID=$(dfx canister id e2e_project_frontend) + rm .dfx/local/canister_ids.json + assert_command dfx canister stop "${FRONTEND_ID}" + assert_command dfx canister delete "${FRONTEND_ID}" + assert_command dfx cycles balance + assert_eq "22.379 TC (trillion cycles)." +} \ No newline at end of file diff --git a/e2e/tests-dfx/deps.bash b/e2e/tests-dfx/deps.bash index 540aa1329a..c56aa217be 100644 --- a/e2e/tests-dfx/deps.bash +++ b/e2e/tests-dfx/deps.bash @@ -127,7 +127,7 @@ Failed to download from url: http://example.com/c.wasm." cd ../onchain dfx canister stop a - dfx canister delete a + dfx canister delete a --no-withdrawal cd ../app assert_command_fail dfx deps pull --network local @@ -150,8 +150,6 @@ Failed to download from url: http://example.com/c.wasm." setup_onchain - # TODO: test gzipped wasm can be pulled when we have "gzip" option in dfx.json (SDK-1102) - # pull canisters in app project cd app assert_file_not_exists "deps/pulled.json" @@ -180,17 +178,16 @@ Failed to download from url: http://example.com/c.wasm." assert_command dfx deps pull --network local -vvv assert_contains "The canister wasm was found in the cache." # cache hit - # sad path 1: wasm hash doesn't match on chain + # warning: hash mismatch rm -r "${PULLED_DIR:?}/" cd ../onchain cp .dfx/local/canisters/c/c.wasm ../www/a.wasm cd ../app - assert_command_fail dfx deps pull --network local - assert_contains "Failed to pull canister $CANISTER_ID_A." - assert_contains "Hash mismatch." + assert_command dfx deps pull --network local + assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download." - # sad path 2: url server doesn't have the file + # sad path: url server doesn't have the file rm -r "${PULLED_DIR:?}/" rm ../www/a.wasm @@ -199,8 +196,7 @@ Failed to download from url: http://example.com/c.wasm." assert_contains "Failed to download from url:" } - -@test "dfx deps pull can check hash when dfx:wasm_hash specified" { +@test "dfx deps pull works when wasm_hash or wasm_hash_url specified" { use_test_specific_cache_root # dfx deps pull will download files to cache # start a "mainnet" replica which host the onchain canisters @@ -228,11 +224,20 @@ Failed to download from url: http://example.com/c.wasm." cp .dfx/local/canisters/b/b.wasm.gz ../www/b.wasm.gz cp .dfx/local/canisters/c/c.wasm ../www/c.wasm - CUSTOM_HASH="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)" - jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH"'"' dfx.json | sponge dfx.json - dfx build a # .dfx/local/canisters/a/a.wasm is replaced. The new wasm has wasm_hash defined and will be installed. + # A: set dfx:wasm_hash + CUSTOM_HASH_A="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)" + jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH_A"'"' dfx.json | sponge dfx.json + # B: set dfx:wasm_hash_url + echo -n "$(sha256sum .dfx/local/canisters/b/b.wasm.gz | cut -d " " -f 1)" > ../www/b.wasm.gz.sha256 + jq '.canisters.b.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/b.wasm.gz.sha256"'"' dfx.json | sponge dfx.json + # C: set both dfx:wasm_hash and dfx:wasm_hash_url. This should be avoided by providers. + CUSTOM_HASH_C="$(sha256sum .dfx/local/canisters/c/c.wasm | cut -d " " -f 1)" + jq '.canisters.c.pullable.wasm_hash="'"$CUSTOM_HASH_C"'"' dfx.json | sponge dfx.json + echo -n "$CUSTOM_HASH_C" > ../www/c.wasm.sha256 + jq '.canisters.c.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/c.wasm.sha256"'"' dfx.json | sponge dfx.json + + dfx build - # cd ../../../ dfx canister install a --argument 1 dfx canister install b dfx canister install c --argument 3 @@ -243,18 +248,20 @@ Failed to download from url: http://example.com/c.wasm." assert_command dfx deps pull --network local -vvv assert_contains "Canister $CANISTER_ID_A specified a custom hash:" + assert_contains "Canister $CANISTER_ID_B specified a custom hash via url:" + assert_contains "WARN: Canister $CANISTER_ID_C specified both \`wasm_hash\` and \`wasm_hash_url\`. \`wasm_hash\` will be used." + assert_contains "Canister $CANISTER_ID_C specified a custom hash:" - # error case: hash mismatch + # warning: hash mismatch PULLED_DIR="$DFX_CACHE_ROOT/.cache/dfinity/pulled/" rm -r "${PULLED_DIR:?}/" cd ../onchain cp .dfx/local/canisters/a/a.wasm ../www/a.wasm # now the webserver has the onchain version of canister_a which won't match wasm_hash cd ../app - assert_command_fail dfx deps pull --network local -vvv + assert_command dfx deps pull --network local -vvv assert_contains "Canister $CANISTER_ID_A specified a custom hash:" - assert_contains "Failed to pull canister $CANISTER_ID_A." - assert_contains "Hash mismatch." + assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download." } @test "dfx deps init works" { @@ -325,11 +332,11 @@ candid:args => (nat)" # delete onchain canisters so that the replica has no canisters as a clean local replica cd ../onchain dfx canister stop a - dfx canister delete a + dfx canister delete a --no-withdrawal dfx canister stop b - dfx canister delete b + dfx canister delete b --no-withdrawal dfx canister stop c - dfx canister delete c + dfx canister delete c --no-withdrawal cd ../app assert_command dfx deps init # b is set here @@ -355,10 +362,10 @@ Installing canister: $CANISTER_ID_C (dep_c)" # deployed pull dependencies can be stopped and deleted assert_command dfx canister stop dep_b --identity anonymous - assert_command dfx canister delete dep_b --identity anonymous + assert_command dfx canister delete dep_b --identity anonymous --no-withdrawal assert_command dfx canister stop $CANISTER_ID_A --identity anonymous - assert_command dfx canister delete $CANISTER_ID_A --identity anonymous + assert_command dfx canister delete $CANISTER_ID_A --identity anonymous --no-withdrawal # error cases ## set wrong init argument @@ -397,11 +404,11 @@ Installing canister: $CANISTER_ID_C (dep_c)" # delete onchain canisters so that the replica has no canisters as a clean local replica cd ../onchain dfx canister stop a - dfx canister delete a + dfx canister delete a --no-withdrawal dfx canister stop b - dfx canister delete b + dfx canister delete b --no-withdrawal dfx canister stop c - dfx canister delete c + dfx canister delete c --no-withdrawal cd ../app assert_command_fail dfx canister create dep_b @@ -434,7 +441,7 @@ Installing canister: $CANISTER_ID_C (dep_c)" # start a clean local replica dfx canister stop app - dfx canister delete app + dfx canister delete app --no-withdrawal assert_command dfx deploy # only deploy app canister } @@ -443,8 +450,8 @@ Installing canister: $CANISTER_ID_C (dep_c)" # verify the help message assert_command dfx deps pull -h - assert_contains "Pull canisters upon which the project depends. This command connects to the \"ic\" mainnet by default. -You can still choose other network by setting \`--network\`" + assert_contains "Pull canisters upon which the project depends. This command connects to the \"ic\" mainnet by default." + assert_contains "You can still choose other network by setting \`--network\`" assert_command dfx deps pull assert_contains "There are no pull dependencies defined in dfx.json" diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index 6841792b95..f2ebd389cb 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -169,7 +169,15 @@ pub struct Pullable { /// # wasm_hash /// SHA256 hash of the wasm module located at wasm_url. /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. + /// The hash can also be specified via a URL using the `wasm_hash_url` field. + /// If both are defined, the `wasm_hash_url` field will be ignored. pub wasm_hash: Option, + /// # wasm_hash_url + /// Specify the SHA256 hash of the wasm module via this URL. + /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. + /// The hash can also be specified directly using the `wasm_hash` field. + /// If both are defined, the `wasm_hash_url` field will be ignored. + pub wasm_hash_url: Option, /// # dependencies /// Canister IDs (Principal) of direct dependencies. #[schemars(with = "Vec::")] diff --git a/src/dfx-core/src/config/model/local_server_descriptor.rs b/src/dfx-core/src/config/model/local_server_descriptor.rs index 6d59829cfa..871da03dc7 100644 --- a/src/dfx-core/src/config/model/local_server_descriptor.rs +++ b/src/dfx-core/src/config/model/local_server_descriptor.rs @@ -98,11 +98,6 @@ impl LocalServerDescriptor { self.data_directory.join("icx-proxy-pid") } - /// This file contains the listening port of the ic-ref process - pub fn ic_ref_port_path(&self) -> PathBuf { - self.data_directory.join("ic-ref.port") - } - /// This file contains the pid of the ic-btc-adapter process pub fn btc_adapter_pid_path(&self) -> PathBuf { self.data_directory.join("ic-btc-adapter-pid") @@ -297,12 +292,11 @@ impl LocalServerDescriptor { /// Gets the port of a local replica. /// /// # Prerequisites - /// - A local replica or emulator needs to be running, e.g. with `dfx start`. + /// - A local replica needs to be running, e.g. with `dfx start`. pub fn get_running_replica_port( &self, logger: Option<&Logger>, ) -> Result, NetworkConfigError> { - let emulator_port_path = self.ic_ref_port_path(); let replica_port_path = self.replica_port_path(); match read_port_from(&replica_port_path)? { @@ -312,15 +306,7 @@ impl LocalServerDescriptor { } Ok(Some(port)) } - None => match read_port_from(&emulator_port_path)? { - Some(port) => { - if let Some(logger) = logger { - info!(logger, "Found local emulator running on port {}", port); - } - Ok(Some(port)) - } - None => Ok(self.replica.port), - }, + None => Ok(self.replica.port), } } } diff --git a/src/dfx/src/actors/emulator.rs b/src/dfx/src/actors/emulator.rs deleted file mode 100644 index 241ad760d2..0000000000 --- a/src/dfx/src/actors/emulator.rs +++ /dev/null @@ -1,245 +0,0 @@ -use crate::actors::icx_proxy::signals::{PortReadySignal, PortReadySubscribe}; -use crate::actors::shutdown::{wait_for_child_or_receiver, ChildOrReceiver}; -use crate::actors::shutdown_controller::signals::outbound::Shutdown; -use crate::actors::shutdown_controller::signals::ShutdownSubscribe; -use crate::actors::shutdown_controller::ShutdownController; -use crate::lib::error::{DfxError, DfxResult}; -use actix::{ - Actor, ActorContext, ActorFutureExt, Addr, AsyncContext, Context, Handler, Recipient, - ResponseActFuture, Running, WrapFuture, -}; -use anyhow::bail; -use crossbeam::channel::{unbounded, Receiver, Sender}; -use slog::{debug, info, Logger}; -use std::path::{Path, PathBuf}; -use std::thread::JoinHandle; -use std::time::Duration; - -pub mod signals { - use actix::prelude::*; - - /// A message sent to the Emulator when the process is restarted. Since we're - /// restarting inside our own actor, this message should not be exposed. - #[derive(Message)] - #[rtype(result = "()")] - pub(super) struct EmulatorRestarted { - pub port: u16, - } -} - -/// The configuration for the emulator actor. -#[derive(Clone)] -pub struct Config { - pub ic_ref_path: PathBuf, - pub port: Option, - pub write_port_to: PathBuf, - pub shutdown_controller: Addr, - pub logger: Option, -} - -/// A emulator actor. Starts the emulator, can subscribe to a Ready signal and a -/// Killed signal. -/// This starts a thread that monitors the process and send signals to any subscriber -/// listening for restarts. The message contains the port the emulator is listening to. -/// -/// Signals -/// - PortReadySubscribe -/// Subscribe a recipient (address) to receive a EmulatorReadySignal message when -/// the emulator is ready to listen to a port. The message can be sent multiple -/// times (e.g. if the emulator crashes). -/// If a emulator is already started and another actor sends this message, a -/// EmulatorReadySignal will be sent free of charge in the same thread. -pub struct Emulator { - logger: Logger, - config: Config, - - // We keep the port to send to subscribers on subscription. - port: Option, - stop_sender: Option>, - thread_join: Option>, - - /// Ready Signal subscribers. - ready_subscribers: Vec>, -} - -impl Emulator { - pub fn new(config: Config) -> Self { - let logger = - (config.logger.clone()).unwrap_or_else(|| Logger::root(slog::Discard, slog::o!())); - Emulator { - config, - port: None, - stop_sender: None, - thread_join: None, - ready_subscribers: Vec::new(), - logger, - } - } - - fn wait_for_port_file(file_path: &Path) -> DfxResult { - let mut retries = 0; - loop { - if let Ok(content) = std::fs::read_to_string(file_path) { - if let Ok(port) = content.parse::() { - return Ok(port); - } - } - if retries >= 3000 { - bail!("Cannot start ic-ref: timed out"); - } - std::thread::sleep(Duration::from_millis(100)); - retries += 1; - } - } - - fn start_emulator(&mut self, addr: Addr) -> DfxResult { - let logger = self.logger.clone(); - - let (sender, receiver) = unbounded(); - - let handle = anyhow::Context::context( - emulator_start_thread(logger, self.config.clone(), addr, receiver), - "Failed to start emulator thread.", - )?; - - self.thread_join = Some(handle); - self.stop_sender = Some(sender); - Ok(()) - } - - fn send_ready_signal(&self, port: u16) { - for sub in &self.ready_subscribers { - sub.do_send(PortReadySignal { port }); - } - } -} - -impl Actor for Emulator { - type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - self.start_emulator(ctx.address()) - .expect("Could not start the emulator"); - - self.config - .shutdown_controller - .do_send(ShutdownSubscribe(ctx.address().recipient::())); - } - - fn stopping(&mut self, _ctx: &mut Self::Context) -> Running { - info!(self.logger, "Stopping ic-ref..."); - if let Some(sender) = self.stop_sender.take() { - let _ = sender.send(()); - } - - if let Some(join) = self.thread_join.take() { - let _ = join.join(); - } - - info!(self.logger, "Stopped."); - Running::Stop - } -} - -impl Handler for Emulator { - type Result = (); - - fn handle(&mut self, msg: PortReadySubscribe, _: &mut Self::Context) { - // If we have a port, send that we're already ready! Yeah! - if let Some(port) = self.port { - msg.0.do_send(PortReadySignal { port }); - } - - self.ready_subscribers.push(msg.0); - } -} - -impl Handler for Emulator { - type Result = (); - - fn handle( - &mut self, - msg: signals::EmulatorRestarted, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.port = Some(msg.port); - self.send_ready_signal(msg.port); - } -} - -impl Handler for Emulator { - type Result = ResponseActFuture>; - - fn handle(&mut self, _msg: Shutdown, _ctx: &mut Self::Context) -> Self::Result { - // This is just the example for ResponseActFuture but stopping the context - Box::pin( - async {} - .into_actor(self) // converts future to ActorFuture - .map(|_, _act, ctx| { - ctx.stop(); - Ok(()) - }), - ) - } -} - -fn emulator_start_thread( - logger: Logger, - config: Config, - addr: Addr, - receiver: Receiver<()>, -) -> DfxResult> { - let thread_handler = move || { - // Start the process, then wait for the file. - let ic_ref_path = config.ic_ref_path.as_os_str(); - - // form the ic-start command here similar to emulator command - let mut cmd = std::process::Command::new(ic_ref_path); - match config.port { - Some(port) if port != 0 => cmd.args(["--listen-port", &port.to_string()]), - _ => cmd.args(["--pick-port"]), - }; - cmd.args(["--write-port-to", &config.write_port_to.to_string_lossy()]); - cmd.stdout(std::process::Stdio::inherit()); - cmd.stderr(std::process::Stdio::inherit()); - - loop { - let _ = std::fs::remove_file(&config.write_port_to); - let last_start = std::time::Instant::now(); - debug!(logger, "Starting emulator..."); - let mut child = cmd.spawn().expect("Could not start emulator."); - - let port = Emulator::wait_for_port_file(&config.write_port_to).unwrap(); - addr.do_send(signals::EmulatorRestarted { port }); - - // This waits for the child to stop, or the receiver to receive a message. - // We don't restart the emulator if done = true. - match wait_for_child_or_receiver(&mut child, &receiver) { - ChildOrReceiver::Receiver => { - debug!(logger, "Got signal to stop. Killing emulator process..."); - let _ = child.kill(); - let _ = child.wait(); - break; - } - ChildOrReceiver::Child => { - debug!(logger, "Emulator process failed."); - // If it took less than two seconds to exit, wait a bit before trying again. - if std::time::Instant::now().duration_since(last_start) < Duration::from_secs(2) - { - std::thread::sleep(Duration::from_secs(2)); - } else { - debug!( - logger, - "Last emulator seemed to have been healthy, not waiting..." - ); - } - } - } - } - }; - - std::thread::Builder::new() - .name("emulator-actor".to_owned()) - .spawn(thread_handler) - .map_err(DfxError::from) -} diff --git a/src/dfx/src/actors/mod.rs b/src/dfx/src/actors/mod.rs index ea3d5675c9..0d32bc0e55 100644 --- a/src/dfx/src/actors/mod.rs +++ b/src/dfx/src/actors/mod.rs @@ -1,9 +1,7 @@ -use crate::actors; use crate::actors::btc_adapter::signals::BtcAdapterReadySubscribe; use crate::actors::btc_adapter::BtcAdapter; use crate::actors::canister_http_adapter::signals::CanisterHttpAdapterReadySubscribe; use crate::actors::canister_http_adapter::CanisterHttpAdapter; -use crate::actors::emulator::Emulator; use crate::actors::icx_proxy::signals::PortReadySubscribe; use crate::actors::icx_proxy::{IcxProxy, IcxProxyConfig}; use crate::actors::replica::{BitcoinIntegrationConfig, Replica}; @@ -20,7 +18,6 @@ use std::path::PathBuf; pub mod btc_adapter; pub mod canister_http_adapter; -pub mod emulator; pub mod icx_proxy; pub mod replica; mod shutdown; @@ -82,36 +79,6 @@ pub fn start_canister_http_adapter_actor( Ok(CanisterHttpAdapter::new(actor_config).start().recipient()) } -#[context("Failed to start emulator actor.")] -pub fn start_emulator_actor( - env: &dyn Environment, - local_server_descriptor: &LocalServerDescriptor, - shutdown_controller: Addr, - emulator_port_path: PathBuf, -) -> DfxResult> { - let ic_ref_path = env.get_cache().get_binary_command_path("ic-ref")?; - - // Touch the port file. This ensures it is empty prior to - // handing it over to ic-ref. If we read the file and it has - // contents we shall assume it is due to our spawned ic-ref - // process. - std::fs::write(&emulator_port_path, "").with_context(|| { - format!( - "Failed to write/clear emulator port file {}.", - emulator_port_path.to_string_lossy() - ) - })?; - - let actor_config = actors::emulator::Config { - ic_ref_path, - port: local_server_descriptor.replica.port, - write_port_to: emulator_port_path, - shutdown_controller, - logger: Some(env.get_logger().clone()), - }; - Ok(actors::emulator::Emulator::new(actor_config).start()) -} - #[context("Failed to setup replica environment.")] fn setup_replica_env( local_server_descriptor: &LocalServerDescriptor, diff --git a/src/dfx/src/actors/shutdown.rs b/src/dfx/src/actors/shutdown.rs index 9610c36c83..b33668114d 100644 --- a/src/dfx/src/actors/shutdown.rs +++ b/src/dfx/src/actors/shutdown.rs @@ -17,7 +17,7 @@ pub fn wait_for_child_or_receiver( loop { // Check if either the child exited or a shutdown has been requested. // These can happen in either order in response to Ctrl-C, so increase the chance - // to notice a shutdown request even if the emulator exited quickly. + // to notice a shutdown request even if the replica exited quickly. let child_try_wait = child.try_wait(); let receiver_signalled = receiver.recv_timeout(std::time::Duration::from_millis(100)); diff --git a/src/dfx/src/commands/canister/delete.rs b/src/dfx/src/commands/canister/delete.rs index cb29aff404..af19f9937c 100644 --- a/src/dfx/src/commands/canister/delete.rs +++ b/src/dfx/src/commands/canister/delete.rs @@ -7,10 +7,14 @@ use crate::lib::operations::canister; use crate::lib::operations::canister::{ deposit_cycles, start_canister, stop_canister, update_settings, }; +use crate::lib::operations::cycles_ledger::{ + wallet_deposit_to_cycles_ledger, CYCLES_LEDGER_ENABLED, +}; use crate::lib::root_key::fetch_root_key_if_needed; use crate::util::assets::wallet_wasm; use crate::util::blob_from_arguments; -use anyhow::Context; +use crate::util::clap::parsers::icrc_subaccount_parser; +use anyhow::{bail, Context}; use candid::Principal; use clap::Parser; use dfx_core::canister::build_wallet_canister; @@ -23,6 +27,7 @@ use ic_utils::interfaces::management_canister::builders::InstallMode; use ic_utils::interfaces::management_canister::CanisterStatus; use ic_utils::interfaces::ManagementCanister; use ic_utils::Argument; +use icrc_ledger_types::icrc1::account::{Account, Subaccount}; use num_traits::cast::ToPrimitive; use slog::info; use std::convert::TryFrom; @@ -71,6 +76,11 @@ pub struct CanisterDeleteOpts { /// Auto-confirm deletion for a non-stopped canister. #[arg(long, short)] yes: bool, + + /// Subaccount of the selected identity to deposit cycles to. + //TODO(SDK-1331): unhide + #[arg(long, value_parser = icrc_subaccount_parser, hide = true)] + to_subaccount: Option, } #[context("Failed to delete canister '{}'.", canister)] @@ -83,6 +93,7 @@ async fn delete_canister( withdraw_cycles_to_canister: Option, withdraw_cycles_to_dank: bool, withdraw_cycles_to_dank_principal: Option, + to_cycles_ledger_subaccount: Option, ) -> DfxResult { let log = env.get_logger(); let mut canister_id_store = env.get_canister_id_store()?; @@ -99,19 +110,23 @@ async fn delete_canister( let to_dank = withdraw_cycles_to_dank || withdraw_cycles_to_dank_principal.is_some(); // Get the canister to transfer the cycles to. - let target_canister_id = if no_withdrawal { - None + let withdraw_target = if no_withdrawal { + WithdrawTarget::NoWithdrawal } else if to_dank { - Some(DANK_PRINCIPAL) + WithdrawTarget::Dank } else { match withdraw_cycles_to_canister { Some(ref target_canister_id) => { - Some(Principal::from_text(target_canister_id).with_context(|| { - format!("Failed to read canister id {:?}.", target_canister_id) - })?) + let canister_id = + Principal::from_text(target_canister_id).with_context(|| { + format!("Failed to read canister id {:?}.", target_canister_id) + })?; + WithdrawTarget::Canister { canister_id } } None => match call_sender { - CallSender::Wallet(wallet_id) => Some(*wallet_id), + CallSender::Wallet(wallet_id) => WithdrawTarget::Canister { + canister_id: *wallet_id, + }, CallSender::SelectedId => { let network = env.get_network_descriptor(); let agent_env = create_agent_environment(env, Some(network.name.clone()))?; @@ -120,7 +135,19 @@ async fn delete_canister( .expect("No selected identity.") .to_string(); // If there is no wallet, then do not attempt to withdraw the cycles. - wallet_canister_id(network, &identity_name)? + match wallet_canister_id(network, &identity_name)? { + Some(canister_id) => WithdrawTarget::Canister { canister_id }, + None if CYCLES_LEDGER_ENABLED => { + let Some(my_principal) = env.get_selected_identity_principal() else { bail!("Identity has no principal attached") }; + WithdrawTarget::CyclesLedger { + to: Account { + owner: my_principal, + subaccount: to_cycles_ledger_subaccount, + }, + } + } + _ => WithdrawTarget::NoWithdrawal, + } } }, } @@ -135,11 +162,10 @@ async fn delete_canister( }; fetch_root_key_if_needed(env).await?; - if let Some(target_canister_id) = target_canister_id { + if withdraw_target != WithdrawTarget::NoWithdrawal { info!( log, - "Beginning withdrawal of cycles to canister {}; on failure try --no-wallet --no-withdrawal.", - target_canister_id + "Beginning withdrawal of cycles; on failure try --no-wallet --no-withdrawal." ); // Determine how many cycles we can withdraw. @@ -197,40 +223,55 @@ async fn delete_canister( break; } let cycles_to_withdraw = cycles - margin; - let result = if !to_dank { - info!( - log, - "Attempting to transfer {} cycles to canister {}.", - cycles_to_withdraw, - target_canister_id - ); - // Transfer cycles from the source canister to the target canister using the temporary wallet. - deposit_cycles( - env, - target_canister_id, - &CallSender::Wallet(canister_id), - cycles_to_withdraw, - ) - .await - } else { - info!( - log, - "Attempting to transfer {} cycles to dank principal {}.", - cycles_to_withdraw, - dank_target_principal - ); - let wallet = build_wallet_canister(canister_id, agent).await?; - let opt_principal = Some(dank_target_principal); - wallet - .call( + let result = match withdraw_target { + WithdrawTarget::NoWithdrawal => Ok(()), + WithdrawTarget::Dank => { + info!( + log, + "Attempting to transfer {} cycles to dank principal {}.", + cycles_to_withdraw, + dank_target_principal + ); + let wallet = build_wallet_canister(canister_id, agent).await?; + let opt_principal = Some(dank_target_principal); + wallet + .call( + DANK_PRINCIPAL, + "mint", + Argument::from_candid((opt_principal,)), + cycles_to_withdraw, + ) + .call_and_wait() + .await + .context("Failed mint call.") + } + WithdrawTarget::Canister { + canister_id: target_canister_id, + } => { + info!( + log, + "Attempting to transfer {} cycles to canister {}.", + cycles_to_withdraw, + target_canister_id + ); + // Transfer cycles from the source canister to the target canister using the temporary wallet. + deposit_cycles( + env, target_canister_id, - "mint", - Argument::from_candid((opt_principal,)), + &CallSender::Wallet(canister_id), cycles_to_withdraw, ) - .call_and_wait() .await - .context("Failed mint call.") + } + WithdrawTarget::CyclesLedger { to } => { + wallet_deposit_to_cycles_ledger( + agent, + canister_id, + cycles_to_withdraw, + to, + ) + .await + } }; if result.is_ok() { info!(log, "Successfully withdrew {} cycles.", cycles_to_withdraw); @@ -239,7 +280,7 @@ async fn delete_canister( info!(log, "Not enough margin. Trying again with more margin."); attempts += 1; } else { - // Unforseen error. Report it back to user + // Unforeseen error. Report it back to user result?; } } @@ -293,6 +334,7 @@ pub async fn exec( opts.withdraw_cycles_to_canister, opts.withdraw_cycles_to_dank, opts.withdraw_cycles_to_dank_principal, + opts.to_subaccount, ) .await } else if opts.all { @@ -307,6 +349,7 @@ pub async fn exec( opts.withdraw_cycles_to_canister.clone(), opts.withdraw_cycles_to_dank, opts.withdraw_cycles_to_dank_principal.clone(), + opts.to_subaccount, ) .await?; } @@ -316,3 +359,11 @@ pub async fn exec( unreachable!() } } + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum WithdrawTarget { + NoWithdrawal, + Dank, + CyclesLedger { to: Account }, + Canister { canister_id: Principal }, +} diff --git a/src/dfx/src/commands/deps/pull.rs b/src/dfx/src/commands/deps/pull.rs index 1f53471207..db92b3cb6f 100644 --- a/src/dfx/src/commands/deps/pull.rs +++ b/src/dfx/src/commands/deps/pull.rs @@ -16,12 +16,13 @@ use crate::util::download_file; use anyhow::{anyhow, bail, Context}; use candid::Principal; use clap::Parser; +use dfx_core::config::model::dfinity::Pullable; use dfx_core::fs::composite::{ensure_dir_exists, ensure_parent_dir_exists}; use fn_error_context::context; use ic_agent::{Agent, AgentError}; use ic_wasm::metadata::get_metadata; use sha2::{Digest, Sha256}; -use slog::{error, info, trace, Logger}; +use slog::{error, info, trace, warn, Logger}; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::io::Write; use std::path::Path; @@ -150,24 +151,7 @@ async fn download_and_generate_pulled_canister( let dfx_metadata = fetch_dfx_metadata(agent, &canister_id).await?; let pullable = dfx_metadata.get_pullable()?; - // lookup `wasm_hash` in dfx metadata. If not available, get the hash of the on chain canister. - let hash_on_chain = match &pullable.wasm_hash { - Some(wasm_hash_str) => { - trace!( - logger, - "Canister {canister_id} specified a custom hash: {wasm_hash_str}" - ); - hex::decode(wasm_hash_str)? - } - None => { - match read_state_tree_canister_module_hash(agent, canister_id).await? { - Some(hash_on_chain) => hash_on_chain, - None => { - bail!("Canister {canister_id} doesn't have module hash. Perhaps it's not installed."); - } - } - } - }; + let hash_on_chain = get_hash_on_chain(agent, logger, canister_id, pullable).await?; pulled_canister.wasm_hash = hex::encode(&hash_on_chain); @@ -205,10 +189,12 @@ async fn download_and_generate_pulled_canister( // hash check let hash_download = Sha256::digest(&content); if hash_download.as_slice() != hash_on_chain { - bail!( - "Hash mismatch. + warn!( + logger, + "Canister {} has different hash between on chain and download. on chain: {} download: {}", + canister_id, hex::encode(hash_on_chain), hex::encode(hash_download.as_slice()) ); @@ -289,6 +275,52 @@ async fn fetch_metadata( } } +// Get expected hash of the canister wasm. +// If `wasm_hash` is specified in dfx metadata, use it. +// If `wasm_hash_url` is specified in dfx metadata, download the hash from the url. +// Otherwise, get the hash of the on chain canister. +async fn get_hash_on_chain( + agent: &Agent, + logger: &Logger, + canister_id: Principal, + pullable: &Pullable, +) -> DfxResult> { + if pullable.wasm_hash.is_some() && pullable.wasm_hash_url.is_some() { + warn!(logger, "Canister {canister_id} specified both `wasm_hash` and `wasm_hash_url`. `wasm_hash` will be used."); + }; + if let Some(wasm_hash_str) = &pullable.wasm_hash { + trace!( + logger, + "Canister {canister_id} specified a custom hash: {wasm_hash_str}" + ); + Ok(hex::decode(wasm_hash_str) + .with_context(|| format!("Failed to decode {wasm_hash_str} as sha256 hash."))?) + } else if let Some(wasm_hash_url) = &pullable.wasm_hash_url { + trace!( + logger, + "Canister {canister_id} specified a custom hash via url: {wasm_hash_url}" + ); + let wasm_hash_url = reqwest::Url::parse(wasm_hash_url) + .with_context(|| format!("{wasm_hash_url} is not a valid URL."))?; + let wasm_hash_content = download_file(&wasm_hash_url) + .await + .with_context(|| format!("Failed to download wasm_hash from {wasm_hash_url}."))?; + let wasm_hash_encoded = String::from_utf8(wasm_hash_content) + .with_context(|| format!("Content from {wasm_hash_url} is not valid text."))?; + Ok(hex::decode(&wasm_hash_encoded) + .with_context(|| format!("Failed to decode {wasm_hash_encoded} as sha256 hash."))?) + } else { + match read_state_tree_canister_module_hash(agent, canister_id).await? { + Some(hash_on_chain) => Ok(hash_on_chain), + None => { + bail!( + "Canister {canister_id} doesn't have module hash. Perhaps it's not installed." + ); + } + } + } +} + #[context("Failed to write to a tempfile then rename it to {}", path.display())] fn write_to_tempfile_then_rename(content: &[u8], path: &Path) -> DfxResult { assert!(path.is_absolute()); diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs index 1fdcd36230..3f0fb5c907 100644 --- a/src/dfx/src/commands/start.rs +++ b/src/dfx/src/commands/start.rs @@ -1,8 +1,8 @@ use crate::actors::icx_proxy::signals::PortReadySubscribe; use crate::actors::icx_proxy::IcxProxyConfig; use crate::actors::{ - start_btc_adapter_actor, start_canister_http_adapter_actor, start_emulator_actor, - start_icx_proxy_actor, start_replica_actor, start_shutdown_controller, + start_btc_adapter_actor, start_canister_http_adapter_actor, start_icx_proxy_actor, + start_replica_actor, start_shutdown_controller, }; use crate::config::dfx_version_str; use crate::error_invalid_argument; @@ -53,24 +53,20 @@ pub struct StartOpts { #[arg(long)] clean: bool, - /// Runs a dedicated emulator instead of the replica - #[arg(long)] - emulator: bool, - /// Address of bitcoind node. Implies --enable-bitcoin. - #[arg(long, conflicts_with("emulator"), action = ArgAction::Append)] + #[arg(long, action = ArgAction::Append)] bitcoin_node: Vec, /// enable bitcoin integration - #[arg(long, conflicts_with("emulator"))] + #[arg(long)] enable_bitcoin: bool, /// enable canister http requests - #[arg(long, conflicts_with("emulator"))] + #[arg(long)] enable_canister_http: bool, /// The delay (in milliseconds) an update call should take. Lower values may be expedient in CI. - #[arg(long, conflicts_with("emulator"), default_value_t = 600)] + #[arg(long, default_value_t = 600)] artificial_delay: u32, /// Start even if the network config was modified. @@ -142,7 +138,6 @@ pub fn exec( StartOpts { host, background, - emulator, clean, force, bitcoin_node, @@ -183,7 +178,6 @@ pub fn exec( enable_bitcoin, bitcoin_node, enable_canister_http, - emulator, domain, )?; @@ -244,7 +238,6 @@ pub fn exec( })?; let replica_port_path = empty_writable_path(local_server_descriptor.replica_port_path())?; - let emulator_port_path = empty_writable_path(local_server_descriptor.ic_ref_port_path())?; if background { send_background()?; @@ -328,11 +321,8 @@ pub fn exec( replica_config }; - let effective_config = if emulator { - CachedConfig::emulator() - } else { - CachedConfig::replica(&replica_config) - }; + let effective_config = CachedConfig::replica(&replica_config); + if !clean && !force && previous_config_path.exists() { let previous_config = load_json_file(&previous_config_path) .context("Failed to read replica configuration. Rerun with `--clean`.")?; @@ -349,15 +339,7 @@ pub fn exec( let _proxy = system.block_on(async move { let shutdown_controller = start_shutdown_controller(env)?; - let port_ready_subscribe: Recipient = if emulator { - let emulator = start_emulator_actor( - env, - local_server_descriptor, - shutdown_controller.clone(), - emulator_port_path, - )?; - emulator.recipient() - } else { + let port_ready_subscribe: Recipient = { let btc_adapter_ready_subscribe = btc_adapter_config .map(|btc_adapter_config| { start_btc_adapter_actor( @@ -426,7 +408,6 @@ pub fn exec( #[allow(clippy::large_enum_variant)] pub enum CachedReplicaConfig<'a> { Replica { config: Cow<'a, ReplicaConfig> }, - Emulator, } #[derive(Serialize, Deserialize, PartialEq, Eq)] @@ -445,12 +426,6 @@ impl<'a> CachedConfig<'a> { }, } } - pub fn emulator() -> Self { - Self { - replica_rev: replica_rev().into(), - config: CachedReplicaConfig::Emulator, - } - } } pub fn apply_command_line_parameters( @@ -461,7 +436,6 @@ pub fn apply_command_line_parameters( enable_bitcoin: bool, bitcoin_nodes: Vec, enable_canister_http: bool, - emulator: bool, domain: Vec, ) -> DfxResult { if enable_canister_http { @@ -472,13 +446,6 @@ pub fn apply_command_line_parameters( warn!(logger, "Canister HTTP suppport is enabled by default. It can be disabled through dfx.json or networks.json."); } - if emulator { - warn!( - logger, - "The --emulator parameter is deprecated and will be discontinued soon." - ); - } - let _ = network_descriptor.local_server_descriptor()?; let mut local_server_descriptor = network_descriptor.local_server_descriptor.unwrap(); diff --git a/src/dfx/src/lib/operations/cycles_ledger.rs b/src/dfx/src/lib/operations/cycles_ledger.rs index cc2cb45cce..265e476323 100644 --- a/src/dfx/src/lib/operations/cycles_ledger.rs +++ b/src/dfx/src/lib/operations/cycles_ledger.rs @@ -9,17 +9,18 @@ use crate::lib::operations::canister::create_canister::{ CANISTER_CREATE_FEE, CANISTER_INITIAL_CYCLE_BALANCE, }; use crate::lib::retryable::retryable; -use anyhow::{anyhow, bail}; +use anyhow::{anyhow, bail, Context}; use backoff::future::retry; use backoff::ExponentialBackoff; use candid::{CandidType, Decode, Encode, Nat, Principal}; +use dfx_core::canister::build_wallet_canister; use fn_error_context::context; use ic_agent::Agent; use ic_utils::call::SyncCall; use ic_utils::interfaces::management_canister::builders::CanisterSettings; -use ic_utils::Canister; +use ic_utils::{Argument, Canister}; use icrc_ledger_types::icrc1; -use icrc_ledger_types::icrc1::account::Subaccount; +use icrc_ledger_types::icrc1::account::{Account, Subaccount}; use icrc_ledger_types::icrc1::transfer::{BlockIndex, TransferError}; use serde::Deserialize; use slog::{info, Logger}; @@ -33,6 +34,7 @@ const ICRC1_BALANCE_OF_METHOD: &str = "icrc1_balance_of"; const ICRC1_TRANSFER_METHOD: &str = "icrc1_transfer"; const SEND_METHOD: &str = "send"; const CREATE_CANISTER_METHOD: &str = "create_canister"; +const CYCLES_LEDGER_DEPOSIT_METHOD: &str = "deposit"; const CYCLES_LEDGER_CANISTER_ID: Principal = Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x02, 0x01, 0x01]); @@ -320,6 +322,32 @@ pub async fn create_with_cycles_ledger( } } +pub async fn wallet_deposit_to_cycles_ledger( + agent: &Agent, + wallet_id: Principal, + cycles_to_withdraw: u128, + to: Account, +) -> DfxResult { + // TODO(FI-1022): Import types from cycles ledger crate once available + #[derive(CandidType)] + pub struct DepositArg { + pub to: Account, + pub memo: Option>, + } + + build_wallet_canister(wallet_id, agent) + .await? + .call128( + CYCLES_LEDGER_CANISTER_ID, + CYCLES_LEDGER_DEPOSIT_METHOD, + Argument::from_candid((DepositArg { to, memo: None },)), + cycles_to_withdraw, + ) + .call_and_wait() + .await + .context("Failed deposit call.") +} + #[test] fn ledger_canister_id_text_representation() { assert_eq!( diff --git a/src/dfx/src/lib/replica/status.rs b/src/dfx/src/lib/replica/status.rs index 3efc7ee84f..9b896aa045 100644 --- a/src/dfx/src/lib/replica/status.rs +++ b/src/dfx/src/lib/replica/status.rs @@ -17,12 +17,7 @@ pub async fn ping_and_wait(url: &str) -> DfxResult { let status = agent.status().await; match status { Ok(status) => { - let healthy = match &status.replica_health_status { - Some(status) if status == "healthy" => true, - None => true, // emulator doesn't report replica_health_status - _ => false, - }; - if healthy { + if matches!(&status.replica_health_status, Some(status) if status == "healthy") { break; } }