xtask: migrating from duct to xshell

xshell does not handle any async, so it also required the addition of tokio

Cargo.toml

@@ -42,7 +42,7 @@ futures = { version = "0.3.29" }
 jsonrpsee = { version = "0.20.3" }
 tracing = { version = "0.1.40" }
 tracing-subscriber = { version = "0.3.18" }
-tokio = { version = "1.34.0" }
+tokio = { version = "1.36.0" }
 async-trait = { version = "0.1.74" }
 fex = { version = "0.4.3" }
 hex-literal = { version = "0.4.1" }
@@ -181,5 +181,6 @@ pallet-ikura-blobs = { path = "ikura/chain/pallets/blobs", default-features = fa
 pallet-ikura-length-fee-adjustment = { path = "ikura/chain/pallets/length-fee-adjustment", default-features = false }
 
 # xtask
-duct = { version = "0.13.7" }
+xshell = { version = "0.2.5" }
+nix = { version = "0.27.1" }
 ctrlc = { version = "3.4.2" }

xtask/Cargo.toml

@@ -10,9 +10,11 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-duct = { workspace = true }
+xshell = { workspace = true }
+nix = { workspace = true, features = ["signal", "process"] }
+tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread", "time", "process", "sync", "signal"] }
 clap = { workspace = true, features = ["derive"] }
 anyhow = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
-ctrlc = { workspace = true }
+serde_json = { workspace = true }

xtask/src/build.rs

@@ -1,44 +1,42 @@
-use crate::{cli::BuildParams, logging::create_with_logs};
-use duct::cmd;
+use crate::{cli::BuildParams, logging::create_log_file, run_with_logs};
 
 // TODO: https://github.com/thrumdev/blobs/issues/225
 
-pub fn build(project_path: &std::path::Path, params: BuildParams) -> anyhow::Result<()> {
+pub async fn build(project_path: &std::path::Path, params: BuildParams) -> anyhow::Result<()> {
     if params.skip {
         return Ok(());
     }
 
+    let sh = xshell::Shell::new()?;
+
     tracing::info!("Building logs redirected {}", params.log_path);
-    let with_logs = create_with_logs(project_path, params.log_path);
+    let log_path = dbg!(create_log_file(project_path, &params.log_path));
 
     // `it is advisable to use CARGO environmental variable to get the right cargo`
     // quoted by xtask readme
     let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
 
-    with_logs(
+    run_with_logs!(
         "Building ikura-node",
-        cmd!(&cargo, "build", "-p", "ikura-node", "--release"),
-    )
-    .run()?;
+        sh,
+        "{cargo} build -p ikura-node --release",
+        log_path
+    )?;
 
-    with_logs(
+    run_with_logs!(
         "Building ikura-shim",
-        cmd!(&cargo, "build", "-p", "ikura-shim", "--release"),
-    )
-    .run()?;
+        sh,
+        "{cargo} build -p ikura-node --release",
+        log_path
+    )?;
 
-    let sov_demo_rollup_path = project_path.join("demo/sovereign/demo-rollup/");
-    #[rustfmt::skip]
-    with_logs(
+    sh.change_dir(project_path.join("demo/sovereign/demo-rollup/"));
+    run_with_logs!(
         "Building sovereign demo-rollup",
-        cmd!(
-            "sh", "-c",
-            format!(
-                "cd {}  && {cargo} build --release",
-                sov_demo_rollup_path.to_string_lossy()
-            )
-        ),
-    ).run()?;
+        sh,
+        "{cargo} build --release",
+        log_path
+    )?;
 
     Ok(())
 }

xtask/src/logging.rs

@@ -1,65 +1,100 @@
-use std::path::Path;
-use std::{io::Write, path::PathBuf};
-use tracing::{info, warn};
+use std::path::{Path, PathBuf};
+use tracing::warn;
 
 // If log_path is relative it will be made absolute relative to the project_path
 //
 // The absolute path of where the log file is created is returned
-fn create_log_file(project_path: &Path, log_path: &String) -> std::io::Result<PathBuf> {
+pub fn create_log_file(project_path: &Path, log_path: &String) -> Option<String> {
     let mut log_path: PathBuf = Path::new(&log_path).to_path_buf();
 
     if log_path.is_relative() {
         log_path = project_path.join(log_path);
     }
 
     if let Some(prefix) = log_path.parent() {
-        std::fs::create_dir_all(prefix)?;
+        std::fs::create_dir_all(prefix)
+            .map_err(|e| warn!("Impossible redirect logs, using stdout instead. Error: {e}"))
+            .ok()?;
     }
-    std::fs::File::create(&log_path)?;
-    Ok(log_path)
+
+    std::fs::File::create(&log_path)
+        .map_err(|e| warn!("Impossible redirect logs, using stdout instead. Error: {e}"))
+        .ok()?;
+    Some(log_path.to_string_lossy().to_string())
 }
 
-// If the log file cannot be created due to any reasons,
-// such as lack of permission to create files or new folders in the path,
-// things will be printed to stdout instead of being redirected to the logs file
+// This macro will accept a description of the command, an xshell::Shell, a command
+// (a string on which format will be called), the log_path, and optionally the process group
+// used to create the command.
 //
-// The returned closure will accept a description of the command and the command itself as a duct::Expression.
-// The description will be printed to both stdout and the log file, if possible, while
-// to the expression will be added the redirection of the logs, if possible.
-pub fn create_with_logs(
-    project_path: &Path,
-    log_path: String,
-) -> Box<dyn Fn(&str, duct::Expression) -> duct::Expression> {
-    let without_logs = |description: &str, cmd: duct::Expression| -> duct::Expression {
-        info!("{description}");
-        cmd
-    };
-
-    let log_path = match create_log_file(project_path, &log_path) {
-        Ok(log_path) => log_path,
-        Err(e) => {
-            warn!("Impossible redirect logs, using stdout instead. Error: {e}");
-            return Box::new(without_logs);
-        }
-    };
+// The description will be logged with the info log level, and the command will be created
+// using the provided shell and converted to a tokio::process::Command to redirect stdout and stderr.
+//
+// If log is None, then things will be redirected to stdout.
+// Instead, if it contains a path, it will be used to redirect the command's output.
+//
+// The provided path needs to be a valid path, `create_log_file` should be used before use this macro
+#[macro_export]
+macro_rules! cmd_with_logs {
+    ($description:expr, $sh:expr, $cmd:literal, $log:expr $(,$gpid:literal)?) => {{
+        use std::process::Stdio;
+        tracing::info!("{}", $description);
+        let (stdout, stderr) = match $log {
+            None => (Stdio::inherit(), Stdio::inherit()),
+            Some(ref log_path) => {
+                // redirecting stdout and stderr into log_path
+                let mut log_out_file = std::fs::File::options()
+                    .append(true)
+                    .create(true)
+                    .open(&log_path)
+                    .expect("Log file does not exist");
+                let log_err_file = log_out_file.try_clone()?;
 
-    let with_logs = move |description: &str, cmd: duct::Expression| -> duct::Expression {
-        // The file has just been created
-        let mut log_file = std::fs::File::options()
-            .append(true)
-            .open(&log_path)
-            .unwrap();
+                use std::io::Write;
+                let _ = log_out_file
+                    .write(format!("{}\n", $description).as_bytes())
+                    .map_err(|e| tracing::warn!("Error writing into {log_path}, error: {e}"));
+                let _ = log_out_file
+                    .flush()
+                    .map_err(|e| tracing::warn!("Error writing into {log_path}, error: {e}"));
+                (Stdio::from(log_out_file), Stdio::from(log_err_file))
+            }
+        };
+        #[allow(unused_mut)]
+        let mut std_cmd = std::process::Command::from(xshell::cmd!($sh, $cmd));
+        $(
+            use std::os::unix::process::CommandExt;
+            std_cmd.process_group($gpid);
+        )?
+        tokio::process::Command::from(std_cmd)
+            .stderr(stderr)
+            .stdout(stdout)
+    }};
+}
 
-        info!("{description}");
-        let log_path = log_path.to_string_lossy();
-        let _ = log_file
-            .write(format!("{}\n", description).as_bytes())
-            .map_err(|e| warn!("Error writing into {log_path}, error: {e}",));
-        let _ = log_file
-            .flush()
-            .map_err(|e| warn!("Error writing into {log_path}, error: {e}",));
-        cmd.stderr_to_stdout().stdout_file(log_file)
-    };
+#[macro_export]
+macro_rules! spawn_with_logs {
+    ($description:expr, $sh:expr, $cmd:literal, $log:expr) => {{
+        crate::cmd_with_logs!($description, $sh, $cmd, $log)
+            .kill_on_drop(true)
+            .spawn()
+    }};
+}
 
-    Box::new(with_logs)
+#[macro_export]
+macro_rules! run_with_logs {
+    ($description:expr, $sh:expr, $cmd:literal, $log:expr) => {{
+        let exit_status: anyhow::Result<std::process::ExitStatus> =
+            crate::spawn_with_logs!($description, $sh, $cmd, $log)?
+                .wait()
+                .await
+                .map_err(|e| e.into());
+        match exit_status?.code() {
+            Some(code) if code != 0 => Err(anyhow::anyhow!(
+                "{}, exit with status code: {code}",
+                $description
+            )),
+            _ => Ok(()),
+        }
+    }};
 }