feat: dfx pull can resolve dependencies from on-chain metadata (#2796)

* init pull command

* prepare pullable canisters in e2e test

* reorg

* pull resolve deps

* e2e sad paths

* changelog

* add a step in dummy job

* not run with ic-ref

* fix metadata error

* assert_occurs

* common dep only pulled once

* fix shellcheck

* fix

.github/workflows/build-frontend-canister-dummy.yml

@@ -1,6 +1,6 @@
 name: Check frontend canister build (skipped)
 
-on: 
+on:
   push:
     branches:
       - master

CHANGELOG.md

@@ -23,7 +23,10 @@ This works for both `dfx identity new` and `dfx identity import`.
 
 The flag `--disable-encryption` is deprecated in favour of `--storage-mode plaintext`. It has the same behavior.
 
-### feat: write canister metadata sections for dfx pull
+### feat: dfx pull
+
+- write canister metadata for dfx pull.
+- `dfx pull` can fetch `dfx:deps` metadata and resolve dependencies recursively.
 
 ### fix: dfx deploy --mode reinstall for a single Motoko canister fails to compile
 

e2e/assets/pullable/app/dfx.json

@@ -0,0 +1,21 @@
+{
+    "canisters": {
+        "app": {
+            "type": "custom",
+            "wasm": "",
+            "candid": "",
+            "build": "",
+            "dependencies": [
+                "dep1", "dep2"
+            ]
+        },
+        "dep1": {
+            "type": "pull",
+            "id": "ryjl3-tyaaa-aaaaa-aaaba-cai"
+        },
+        "dep2": {
+            "type": "pull",
+            "id": "r7inp-6aaaa-aaaaa-aaabq-cai"
+        }
+    }
+}

e2e/assets/pullable/onchain/dfx.json

@@ -0,0 +1,22 @@
+{
+  "canisters": {
+    "onchain_a": {
+      "type": "custom",
+      "candid": "src/onchain_a/main.did",
+      "wasm": "src/onchain_a/main.wasm",
+      "build": ""
+    },
+    "onchain_b": {
+      "type": "custom",
+      "candid": "src/onchain_b/main.did",
+      "wasm": "src/onchain_b/main.wasm",
+      "build": ""
+    },
+    "onchain_c": {
+      "type": "custom",
+      "candid": "src/onchain_c/main.did",
+      "wasm": "src/onchain_c/main.wasm",
+      "build": ""
+    }
+  }
+}

e2e/assets/pullable/onchain/src/onchain_a/main.did

@@ -0,0 +1 @@
+service : {}

e2e/assets/pullable/onchain/src/onchain_b/main.did

@@ -0,0 +1 @@
+service : {}

e2e/assets/pullable/onchain/src/onchain_c/main.did

@@ -0,0 +1 @@
+service : {}

e2e/tests-dfx/pull.bash

@@ -34,4 +34,79 @@ teardown() {
 
     assert_command ic-wasm .dfx/local/canisters/e2e_project_backend/e2e_project_backend.wasm metadata dfx:wasm_url
     assert_match "https://example.com/e2e_project.wasm"
-}
+}
+
+@test "dfx pull can resolve dependencies from on-chain canister metadata" {
+    # When ran with ic-ref, got following error:
+    # Certificate is not authorized to respond to queries for this canister. While developing: Did you forget to set effective_canister_id?
+    [ "$USE_IC_REF" ] && skip "skipped for ic-ref"
+    # system-wide local replica
+    dfx_start
+
+    install_asset pullable
+
+    # 1. success path
+    ## 1.1. prepare "onchain" canisters
+    # a -> []
+    # b -> [a]
+    # c -> [a]
+    # app -> [a, b]
+    cd onchain
+
+    echo -n -e \\x00asm\\x01\\x00\\x00\\x00 > src/onchain_a/main.wasm
+
+    echo -n -e \\x00asm\\x01\\x00\\x00\\x00 > src/onchain_b/empty.wasm
+    ic-wasm src/onchain_b/empty.wasm -o src/onchain_b/main.wasm metadata "dfx:deps" -d "onchain_a:rrkah-fqaaa-aaaaa-aaaaq-cai;" -v public
+
+    echo -n -e \\x00asm\\x01\\x00\\x00\\x00 > src/onchain_c/empty.wasm
+    ic-wasm src/onchain_c/empty.wasm -o src/onchain_c/main.wasm metadata "dfx:deps" -d "onchain_a:rrkah-fqaaa-aaaaa-aaaaq-cai;" -v public
+
+    dfx deploy
+
+    assert_command dfx canister metadata ryjl3-tyaaa-aaaaa-aaaba-cai dfx:deps
+    assert_match "onchain_a:rrkah-fqaaa-aaaaa-aaaaq-cai;"
+
+    ## 1.2. pull onchain canisters in "app" project
+    cd ../app
+    assert_command dfx pull dep1
+    assert_match "Pulling canister ryjl3-tyaaa-aaaaa-aaaba-cai...
+Pulling canister rrkah-fqaaa-aaaaa-aaaaq-cai...
+WARN: \`dfx:deps\` metadata not found in canister rrkah-fqaaa-aaaaa-aaaaq-cai."
+
+    assert_command dfx pull # if not specify canister name, all pull type canisters (dep1, dep2) will be pulled
+    assert_match "Pulling canister ryjl3-tyaaa-aaaaa-aaaba-cai...
+Pulling canister r7inp-6aaaa-aaaaa-aaabq-cai...
+Pulling canister rrkah-fqaaa-aaaaa-aaaaq-cai...
+WARN: \`dfx:deps\` metadata not found in canister rrkah-fqaaa-aaaaa-aaaaq-cai."
+    assert_occurs 1 "Pulling canister rrkah-fqaaa-aaaaa-aaaaq-cai..." # common dependency onchain_a is pulled only once
+
+    # 2. sad path: if the canister is not present on-chain
+    cd ../onchain
+    dfx canister uninstall-code onchain_a
+
+    cd ../app
+    assert_command_fail dfx pull
+    assert_contains "Failed while fetch and parse \`dfx:deps\` metadata from canister rrkah-fqaaa-aaaaa-aaaaq-cai."
+    assert_contains "Canister rrkah-fqaaa-aaaaa-aaaaq-cai has no module."
+
+    cd ../onchain
+    dfx canister stop onchain_a
+    dfx canister delete onchain_a
+
+    cd ../app
+    assert_command_fail dfx pull
+    assert_contains "Failed while fetch and parse \`dfx:deps\` metadata from canister rrkah-fqaaa-aaaaa-aaaaq-cai."
+    assert_contains "Canister rrkah-fqaaa-aaaaa-aaaaq-cai not found."
+
+    # 3. sad path: if dependency metadata cannot be read (wrong format)
+    cd ../onchain
+    cd src/onchain_b
+    ic-wasm empty.wasm -o main.wasm metadata "dfx:deps" -d "rrkah-fqaaa-aaaaa-aaaaq-cai;onchain_a" -v public
+    cd ../../ # go back to root of "onchain" project
+    dfx deploy
+
+    cd ../app
+    assert_command_fail dfx pull
+    assert_contains "Failed while fetch and parse \`dfx:deps\` metadata from canister ryjl3-tyaaa-aaaaa-aaaba-cai."
+    assert_contains "Failed to parse \`dfx:deps\` entry: rrkah-fqaaa-aaaaa-aaaaq-cai. Expected \`name:Principal\`."
+}

e2e/utils/assertions.bash

@@ -110,6 +110,29 @@ assert_not_match() {
     fi
 }
 
+# Asserts that a string occurs a number of times in another string.
+# Arguments:
+#    $1 - Expected number of occurance.
+#    $2 - The string to search for.
+#    $3 - The string to search in. By default it will use $output.
+assert_occurs() {
+    expect="$1"
+    search_for="$2"
+    if [[ $# -lt 3 ]]; then
+        search_in="$output"
+    else
+        search_in="$3"
+    fi
+
+    actual="$( echo "$search_in" | grep -o "$search_for" | wc -l | xargs )"
+
+    if [[ "$expect" -ne "$actual" ]]; then
+        batslib_print_kv_single_or_multi 6 "Expect" "$expect" "Actual" "$actual" \
+            | batslib_decorate "Occurrences of \"$search_for\" in \"$search_in\" didn't match expectation." \
+            | fail
+    fi
+}
+
 # Asserts a command will timeout. This assertion will fail if the command finishes before
 # the timeout period. If the command fails, it will also fail.
 # Arguments:

src/dfx/src/commands/canister/metadata.rs

@@ -33,7 +33,12 @@ pub async fn exec(env: &dyn Environment, opts: CanisterMetadataOpts) -> DfxResul
     let metadata = agent
         .read_state_canister_metadata(canister_id, &opts.metadata_name)
         .await
-        .with_context(|| format!("Failed to read controllers of canister {}.", canister_id))?;
+        .with_context(|| {
+            format!(
+                "Failed to read `{}` metadata of canister {}.",
+                opts.metadata_name, canister_id
+            )
+        })?;
 
     stdout().write_all(&metadata)?;
 

src/dfx/src/commands/mod.rs

@@ -20,6 +20,7 @@ mod ledger;
 mod new;
 mod nns;
 mod ping;
+mod pull;
 mod quickstart;
 mod remote;
 mod replica;
@@ -51,6 +52,7 @@ pub enum Command {
     New(new::NewOpts),
     Nns(nns::NnsOpts),
     Ping(ping::PingOpts),
+    Pull(pull::PullOpts),
     Quickstart,
     Remote(remote::RemoteOpts),
     Replica(replica::ReplicaOpts),
@@ -82,6 +84,7 @@ pub fn exec(env: &dyn Environment, cmd: Command) -> DfxResult {
         Command::New(v) => new::exec(env, v),
         Command::Nns(v) => nns::exec(env, v),
         Command::Ping(v) => ping::exec(env, v),
+        Command::Pull(v) => pull::exec(env, v),
         Command::Quickstart => quickstart::exec(env),
         Command::Remote(v) => remote::exec(env, v),
         Command::Replica(v) => replica::exec(env, v),

src/dfx/src/commands/pull.rs

@@ -0,0 +1,116 @@
+use crate::config::dfinity::CanisterTypeProperties;
+use crate::lib::error::DfxResult;
+use crate::lib::root_key::fetch_root_key_if_needed;
+use crate::lib::{environment::Environment, provider::create_agent_environment};
+use crate::NetworkOpt;
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
+
+use anyhow::{anyhow, bail, Context};
+use candid::Principal;
+use clap::Parser;
+use fn_error_context::context;
+use ic_agent::{Agent, AgentError};
+use slog::Logger;
+use tokio::runtime::Runtime;
+
+/// Pings an Internet Computer network and returns its status.
+#[derive(Parser)]
+pub struct PullOpts {
+    /// Specifies the name of the canister you want to pull.
+    /// If you don’t specify a canister name, all pull type canisters defined in the dfx.json file are pulled.
+    canister_name: Option<String>,
+
+    #[clap(flatten)]
+    network: NetworkOpt,
+}
+
+pub fn exec(env: &dyn Environment, opts: PullOpts) -> DfxResult {
+    let agent_env = create_agent_environment(env, opts.network.network)?;
+    let logger = env.get_logger();
+
+    let runtime = Runtime::new().expect("Unable to create a runtime");
+    runtime.block_on(async {
+        fetch_root_key_if_needed(&agent_env).await?;
+
+        let agent = agent_env
+            .get_agent()
+            .ok_or_else(|| anyhow!("Cannot get HTTP client from environment."))?;
+
+        let config = agent_env.get_config_or_anyhow()?;
+        let mut pull_canisters = BTreeMap::new();
+
+        if let Some(map) = &config.get_config().canisters {
+            for (k, v) in map {
+                if let CanisterTypeProperties::Pull { id } = v.type_specific {
+                    pull_canisters.insert(k, id);
+                }
+            }
+        };
+
+        let mut canisters_to_pull: VecDeque<Principal> = match opts.canister_name {
+            Some(s) => match pull_canisters.get(&s) {
+                Some(v) => VecDeque::from([*v]),
+                None => bail!("There is no pull type canister \"{s}\" defined in dfx.json"),
+            },
+            None => pull_canisters.values().cloned().collect(),
+        };
+
+        let mut pulled_canisters: BTreeSet<Principal> = BTreeSet::new();
+
+        while let Some(callee_canister) = canisters_to_pull.pop_front() {
+            if !pulled_canisters.contains(&callee_canister) {
+                pulled_canisters.insert(callee_canister);
+                fetch_deps_to_pull(agent, logger, callee_canister, &mut canisters_to_pull).await?;
+            }
+        }
+
+        Ok(())
+    })
+}
+
+#[context("Failed while fetch and parse `dfx:deps` metadata from canister {canister_id}.")]
+async fn fetch_deps_to_pull(
+    agent: &Agent,
+    logger: &Logger,
+    canister_id: Principal,
+    canisters_to_pull: &mut VecDeque<Principal>,
+) -> DfxResult {
+    slog::info!(logger, "Pulling canister {canister_id}...");
+
+    match agent
+        .read_state_canister_metadata(canister_id, "dfx:deps")
+        .await
+    {
+        Ok(data) => {
+            let data = String::from_utf8(data)?;
+            for entry in data.split_terminator(';') {
+                match entry.split_once(':') {
+                    Some((_, p)) => {
+                        let canister_id = Principal::from_text(p)
+                            .with_context(|| format!("`{p}` is not a valid Principal."))?;
+                        canisters_to_pull.push_back(canister_id);
+                    }
+                    None => bail!(
+                        "Failed to parse `dfx:deps` entry: {entry}. Expected `name:Principal`. "
+                    ),
+                }
+            }
+            Ok(())
+        }
+        Err(agent_error) => match agent_error {
+            AgentError::HttpError(ref e) => {
+                let content = String::from_utf8(e.content.clone())?;
+                if content.starts_with("Custom section") {
+                    slog::warn!(
+                        logger,
+                        "`dfx:deps` metadata not found in canister {canister_id}."
+                    );
+                    Ok(())
+                } else {
+                    Err(anyhow!(agent_error))
+                }
+            }
+            _ => Err(anyhow!(agent_error)),
+        },
+    }
+}