From 2a283a82037cbe67ab7953420caa5c2c098b7cf2 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 22 Jul 2024 14:52:52 +0200 Subject: [PATCH 01/13] Improve directory seach logic for UI tests --- test/test-manager/src/package.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs index f3fd53567386..fbfba1e48db0 100644 --- a/test/test-manager/src/package.rs +++ b/test/test-manager/src/package.rs @@ -34,6 +34,15 @@ pub fn get_app_manifest( .transpose()?; log::info!("App package to upgrade from: {app_package_to_upgrade_from_path:?}"); + // Automatically try to find the UI e2e tests based on the app package + + // Search the specified package folder, or same folder as the app package if missing + let ui_e2e_package_folder = package_folder.unwrap_or( + app_package_path + .parent() + .expect("Path to app package should have parent") + .into(), + ); let capture = VERSION_REGEX .captures(app_package_path.to_str().unwrap()) .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))? @@ -41,7 +50,8 @@ pub fn get_app_manifest( .map(|c| c.as_str()) .expect("Could not parse version from package name: {app_package}"); - let ui_e2e_tests_path = find_app(capture, true, package_type, package_folder.as_ref()).ok(); + let ui_e2e_tests_path = + find_app(capture, true, package_type, Some(&ui_e2e_package_folder)).ok(); log::info!("GUI e2e test binary: {ui_e2e_tests_path:?}"); Ok(Manifest { From b50501b9c2240c09628bf7e63d6ff398faf51f3b Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Thu, 1 Aug 2024 10:31:21 +0200 Subject: [PATCH 02/13] Fix `find_app` file matching bug It could match on dev builds when specifying a stable build. Add back sorting by file name length, which on ok solution to this problem. --- test/test-manager/src/package.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs index fbfba1e48db0..c1f9d0232b4e 100644 --- a/test/test-manager/src/package.rs +++ b/test/test-manager/src/package.rs @@ -1,5 +1,6 @@ use crate::config::{Architecture, OsType, PackageType, VmConfig}; use anyhow::{Context, Result}; +use itertools::Itertools; use once_cell::sync::Lazy; use regex::Regex; use std::path::{Path, PathBuf}; @@ -105,6 +106,7 @@ fn find_app( // Skip for non-Linux, because there's only one package !linux || matching_ident }) // Skip file if it doesn't match the architecture + .sorted_unstable_by_key(|(_path, u8_path)| u8_path.len()) .find(|(_path, u8_path)| u8_path.contains(&app)) // Find match .map(|(path, _)| path).context(if e2e_bin { format!( From 7cff069918f4c412f11f45e098fdf2e22c3ea71b Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 22 Jul 2024 16:41:41 +0200 Subject: [PATCH 03/13] Fix regex on release versions --- test/test-manager/src/package.rs | 46 +++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs index c1f9d0232b4e..4d8741f46629 100644 --- a/test/test-manager/src/package.rs +++ b/test/test-manager/src/package.rs @@ -5,9 +5,6 @@ use once_cell::sync::Lazy; use regex::Regex; use std::path::{Path, PathBuf}; -static VERSION_REGEX: Lazy = - Lazy::new(|| Regex::new(r"\d{4}\.\d+(-beta\d+)?(-dev)?-([0-9a-z])+").unwrap()); - #[derive(Debug, Clone)] pub struct Manifest { pub app_package_path: PathBuf, @@ -44,12 +41,7 @@ pub fn get_app_manifest( .expect("Path to app package should have parent") .into(), ); - let capture = VERSION_REGEX - .captures(app_package_path.to_str().unwrap()) - .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))? - .get(0) - .map(|c| c.as_str()) - .expect("Could not parse version from package name: {app_package}"); + let capture = get_version_from_path(&app_package_path)?; let ui_e2e_tests_path = find_app(capture, true, package_type, Some(&ui_e2e_package_folder)).ok(); @@ -62,6 +54,18 @@ pub fn get_app_manifest( }) } +fn get_version_from_path(app_package_path: &Path) -> Result<&str, anyhow::Error> { + static VERSION_REGEX: Lazy = + Lazy::new(|| Regex::new(r"\d{4}\.\d+((-beta\d+)?(-dev)?-([0-9a-z])+)?").unwrap()); + + VERSION_REGEX + .captures(app_package_path.to_str().unwrap()) + .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))? + .get(0) + .map(|c| c.as_str()) + .context("Could not parse version from package name: {app_package}") +} + fn find_app( app: &str, e2e_bin: bool, @@ -138,3 +142,27 @@ fn get_os_name(package_type: (OsType, Option, Option) OsType::Linux => "linux", } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_version_regex() { + let path = Path::new("../some/path/MullvadVPN-2024.4-beta1-dev-f7df8e_amd64.deb"); + let capture = get_version_from_path(path).unwrap(); + assert_eq!(capture, "2024.4-beta1-dev-f7df8e"); + + let path = Path::new("../some/path/MullvadVPN-2024.4-beta1-f7df8e_amd64.deb"); + let capture = get_version_from_path(path).unwrap(); + assert_eq!(capture, "2024.4-beta1-f7df8e"); + + let path = Path::new("../some/path/MullvadVPN-2024.4-dev-f7df8e_amd64.deb"); + let capture = get_version_from_path(path).unwrap(); + assert_eq!(capture, "2024.4-dev-f7df8e"); + + let path = Path::new("../some/path/MullvadVPN-2024.3_amd64.deb"); + let capture = get_version_from_path(path).unwrap(); + assert_eq!(capture, "2024.3"); + } +} From eb6cdfa95cf55b62a5908e5fcdffdb2fa136559b Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 22 Jul 2024 15:45:27 +0200 Subject: [PATCH 04/13] Change VM config from positional arg to flag --- test/ci-runtests.sh | 2 +- test/test-manager/src/main.rs | 61 +++++++++++++++----------------- test/test-manager/src/summary.rs | 6 ++-- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/test/ci-runtests.sh b/test/ci-runtests.sh index e6b7ba7b18df..0346044e7bdf 100755 --- a/test/ci-runtests.sh +++ b/test/ci-runtests.sh @@ -201,7 +201,7 @@ function run_tests_for_os { --app-package-to-upgrade-from "${prev_filename}" \ --package-folder "$PACKAGES_DIR" \ --test-report "$SCRIPT_DIR/.ci-logs/${os}_report" \ - "$os" 2>&1 | sed "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g" + --vm "$os" 2>&1 | sed "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g" } echo "**********************************" diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs index 79d239f4d0e1..a679745e83b1 100644 --- a/test/test-manager/src/main.rs +++ b/test/test-manager/src/main.rs @@ -28,27 +28,27 @@ struct Args { enum Commands { /// Create or edit a VM config Set { - /// Name of the config - name: String, + /// Name of the VM config + vm: String, /// VM config #[clap(flatten)] config: config::VmConfig, }, - /// Remove specified configuration + /// Remove specified VM config Remove { - /// Name of the config - name: String, + /// Name of the VM config, run `test-manager list` to see available configs + vm: String, }, - /// List available configurations + /// List available VM configurations List, /// Spawn a runner instance without running any tests RunVm { - /// Name of the runner config - name: String, + /// Name of the VM config, run `test-manager list` to see available configs + vm: String, /// Run VNC server on a specified port #[arg(long)] @@ -64,8 +64,9 @@ enum Commands { /// Spawn a runner instance and run tests RunTests { - /// Name of the runner config - name: String, + /// Name of the VM config, run `test-manager list` to see available configs + #[arg(long)] + vm: String, /// Show display of guest #[arg(long, group = "display_args")] @@ -110,8 +111,8 @@ enum Commands { #[arg(long, short)] verbose: bool, - /// Output test results in a structured format. - #[arg(long)] + /// Path to output test results in a structured format + #[arg(long, value_name = "PATH")] test_report: Option, }, @@ -127,7 +128,7 @@ enum Commands { /// to have `provisioner` set to `ssh`, `ssh_user` & `ssh_password` set and /// the `ssh_user` should be able to execute commands with sudo/ as root. Update { - /// Name of the runner config + /// Name of the VM config name: String, }, } @@ -156,34 +157,34 @@ async fn main() -> Result<()> { .context("Failed to load config")?; match args.cmd { Commands::Set { - name, + vm, config: vm_config, - } => vm::set_config(&mut config, &name, vm_config) + } => vm::set_config(&mut config, &vm, vm_config) .await .context("Failed to edit or create VM config"), - Commands::Remove { name } => { - if config.get_vm(&name).is_none() { + Commands::Remove { vm } => { + if config.get_vm(&vm).is_none() { println!("No such configuration"); return Ok(()); } config .edit(|config| { - config.vms.remove_entry(&name); + config.vms.remove_entry(&vm); }) .await .context("Failed to remove config entry")?; - println!("Removed configuration \"{name}\""); + println!("Removed configuration \"{vm}\""); Ok(()) } Commands::List => { println!("Available configurations:"); - for name in config.vms.keys() { - println!("{}", name); + for (vm, config) in config.vms.iter() { + println!("{vm}: {config:#?}"); } Ok(()) } Commands::RunVm { - name, + vm, vnc, keep_changes, } => { @@ -195,9 +196,7 @@ async fn main() -> Result<()> { config::Display::Local }; - let mut instance = vm::run(&config, &name) - .await - .context("Failed to start VM")?; + let mut instance = vm::run(&config, &vm).await.context("Failed to start VM")?; instance.wait().await; @@ -215,7 +214,7 @@ async fn main() -> Result<()> { Ok(()) } Commands::RunTests { - name, + vm, display, vnc, account, @@ -240,12 +239,12 @@ async fn main() -> Result<()> { .unwrap_or(DEFAULT_MULLVAD_HOST.to_owned()); log::debug!("Mullvad host: {mullvad_host}"); - let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?; + let vm_config = vm::get_vm_config(&config, &vm).context("Cannot get VM config")?; let summary_logger = match test_report { Some(path) => Some( summary::SummaryLogger::new( - &name, + &vm, test_rpc::meta::Os::from(vm_config.os_type), &path, ) @@ -263,10 +262,8 @@ async fn main() -> Result<()> { ) .context("Could not find the specified app packages")?; - let mut instance = vm::run(&config, &name) - .await - .context("Failed to start VM")?; - let artifacts_dir = vm::provision(&config, &name, &*instance, &manifest) + let mut instance = vm::run(&config, &vm).await.context("Failed to start VM")?; + let artifacts_dir = vm::provision(&config, &vm, &*instance, &manifest) .await .context("Failed to run provisioning for VM")?; diff --git a/test/test-manager/src/summary.rs b/test/test-manager/src/summary.rs index 30b71948cff8..9706e351a196 100644 --- a/test/test-manager/src/summary.rs +++ b/test/test-manager/src/summary.rs @@ -68,7 +68,7 @@ pub struct SummaryLogger { impl SummaryLogger { /// Create a new logger and log to `path`. If `path` does not exist, it will be created. If it /// already exists, it is truncated and overwritten. - pub async fn new(name: &str, os: Os, path: &Path) -> Result { + pub async fn new(vm: &str, os: Os, path: &Path) -> Result { let mut file = fs::OpenOptions::new() .create(true) .write(true) @@ -77,9 +77,7 @@ impl SummaryLogger { .await .map_err(|err| Error::Open(err, path.to_path_buf()))?; - file.write_all(name.as_bytes()) - .await - .map_err(Error::Write)?; + file.write_all(vm.as_bytes()).await.map_err(Error::Write)?; file.write_u8(b'\n').await.map_err(Error::Write)?; file.write_all(&serde_json::to_vec(&os).map_err(Error::Serialize)?) .await From c4ec5af818910fc18241ea5558828061b4a063a1 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 5 Aug 2024 13:42:53 +0200 Subject: [PATCH 05/13] Add `--gui-package` flag --- test/test-manager/src/main.rs | 10 ++++++++- test/test-manager/src/package.rs | 31 ++++++++++++++++++++------- test/test-manager/src/vm/provision.rs | 12 +++++------ 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs index a679745e83b1..bd9954bc1e63 100644 --- a/test/test-manager/src/main.rs +++ b/test/test-manager/src/main.rs @@ -100,6 +100,12 @@ enum Commands { #[arg(long)] app_package_to_upgrade_from: Option, + /// Package used for GUI tests. Parsed the same way as `--app-package`. + /// If not specified, will look for a package matching the version of the app package. If + /// no such package is found, the GUI tests will fail. + #[arg(long)] + gui_package: Option, + /// Folder to search for packages. Defaults to current directory. #[arg(long, value_name = "DIR")] package_folder: Option, @@ -220,6 +226,7 @@ async fn main() -> Result<()> { account, app_package, app_package_to_upgrade_from, + gui_package, package_folder, test_filters, verbose, @@ -258,6 +265,7 @@ async fn main() -> Result<()> { vm_config, app_package, app_package_to_upgrade_from, + gui_package, package_folder, ) .context("Could not find the specified app packages")?; @@ -290,7 +298,7 @@ async fn main() -> Result<()> { .app_package_to_upgrade_from_path .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()), ui_e2e_tests_filename: manifest - .ui_e2e_tests_path + .gui_package_path .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()), mullvad_host, #[cfg(target_os = "macos")] diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs index 4d8741f46629..b6147d7dd8de 100644 --- a/test/test-manager/src/package.rs +++ b/test/test-manager/src/package.rs @@ -9,7 +9,7 @@ use std::path::{Path, PathBuf}; pub struct Manifest { pub app_package_path: PathBuf, pub app_package_to_upgrade_from_path: Option, - pub ui_e2e_tests_path: Option, + pub gui_package_path: Option, } /// Obtain app packages and their filenames @@ -20,6 +20,7 @@ pub fn get_app_manifest( config: &VmConfig, app_package: String, app_package_to_upgrade_from: Option, + gui_package: Option, package_folder: Option, ) -> Result { let package_type = (config.os_type, config.package_type, config.architecture); @@ -41,20 +42,34 @@ pub fn get_app_manifest( .expect("Path to app package should have parent") .into(), ); - let capture = get_version_from_path(&app_package_path)?; - let ui_e2e_tests_path = - find_app(capture, true, package_type, Some(&ui_e2e_package_folder)).ok(); - log::info!("GUI e2e test binary: {ui_e2e_tests_path:?}"); + let app_version = get_version_from_path(&app_package_path)?; + let gui_package_path = find_app( + match &gui_package { + Some(gui_package) => gui_package, + None => &app_version, + }, + true, + package_type, + Some(&ui_e2e_package_folder), + ); + + // Don't allow the UI/e2e test binary to missing if it's flag was specified + let gui_package_path = match gui_package { + Some(_) => Some(gui_package_path.context("Could not find specified UI/e2e test binary")?), + None => gui_package_path.ok(), + }; + + log::info!("GUI e2e test binary: {gui_package_path:?}"); Ok(Manifest { app_package_path, app_package_to_upgrade_from_path, - ui_e2e_tests_path, + gui_package_path, }) } -fn get_version_from_path(app_package_path: &Path) -> Result<&str, anyhow::Error> { +fn get_version_from_path(app_package_path: &Path) -> Result { static VERSION_REGEX: Lazy = Lazy::new(|| Regex::new(r"\d{4}\.\d+((-beta\d+)?(-dev)?-([0-9a-z])+)?").unwrap()); @@ -62,7 +77,7 @@ fn get_version_from_path(app_package_path: &Path) -> Result<&str, anyhow::Error> .captures(app_package_path.to_str().unwrap()) .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))? .get(0) - .map(|c| c.as_str()) + .map(|c| c.as_str().to_owned()) .context("Could not parse version from package name: {app_package}") } diff --git a/test/test-manager/src/vm/provision.rs b/test/test-manager/src/vm/provision.rs index f440cbe9c81c..42a12eb4261b 100644 --- a/test/test-manager/src/vm/provision.rs +++ b/test/test-manager/src/vm/provision.rs @@ -125,9 +125,9 @@ fn blocking_ssh( } else { log::warn!("No previous app to send to remote") } - if let Some(ui_e2e_tests_path) = &local_app_manifest.ui_e2e_tests_path { - ssh_send_file_path(&session, ui_e2e_tests_path, temp_dir) - .context("Failed to send ui_e2e_tests_path to remote")?; + if let Some(gui_package_path) = &local_app_manifest.gui_package_path { + ssh_send_file_path(&session, gui_package_path, temp_dir) + .context("Failed to send gui_package_path to remote")?; } else { log::warn!("No UI e2e test to send to remote") } @@ -166,13 +166,13 @@ fn blocking_ssh( .app_package_to_upgrade_from_path .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()) .unwrap_or_default(); - let ui_e2e_tests_path = local_app_manifest - .ui_e2e_tests_path + let gui_package_path = local_app_manifest + .gui_package_path .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()) .unwrap_or_default(); let cmd = format!( - "sudo {} {remote_dir} \"{app_package_path}\" \"{app_package_to_upgrade_from_path}\" \"{ui_e2e_tests_path}\"", + "sudo {} {remote_dir} \"{app_package_path}\" \"{app_package_to_upgrade_from_path}\" \"{gui_package_path}\"", dest.display() ); log::debug!("Running setup script on remote, cmd: {cmd}"); From b44cfab6800e7f5acdb852a0397bfe845e8df9a0 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Tue, 30 Jul 2024 12:28:51 +0200 Subject: [PATCH 06/13] Fix nightly lints --- test/test-manager/src/tests/tunnel_state.rs | 6 ++---- test/test-runner/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/test-manager/src/tests/tunnel_state.rs b/test/test-manager/src/tests/tunnel_state.rs index 5a94c60bc691..e8ad102419e9 100644 --- a/test/test-manager/src/tests/tunnel_state.rs +++ b/test/test-manager/src/tests/tunnel_state.rs @@ -440,13 +440,11 @@ pub async fn test_connecting_state_when_corrupted_state_cache( // side-effect of this is that no network traffic is allowed to leak. log::info!("Starting the app back up again"); rpc.start_mullvad_daemon().await?; - wait_for_tunnel_state(mullvad_client.clone(), |state| !state.is_disconnected()) - .await - .map_err(|err| { + wait_for_tunnel_state(mullvad_client.clone(), |state| !state.is_disconnected()).await + .inspect_err(|_| { log::error!("App did not start in an expected state. \ App is not in either `Connecting` or `Connected` state after starting with corrupt state cache! \ There is a possibility of leaks during app startup "); - err })?; log::info!("App successfully recovered from a corrupt tunnel state cache."); diff --git a/test/test-runner/src/main.rs b/test/test-runner/src/main.rs index 3fe91fb72364..59fc83672d77 100644 --- a/test/test-runner/src/main.rs +++ b/test/test-runner/src/main.rs @@ -422,7 +422,7 @@ impl Service for TestServer { match stderr.read_line(&mut line).await { Ok(0) => break, Ok(_) => { - let trimmed = line.trim_end_matches(&['\r', '\n']); + let trimmed = line.trim_end_matches(['\r', '\n']); log::info!("child stderr (pid={pid}): {trimmed}"); line.clear(); } From 4b8330b3d3557882c6a877f0cd3490ed8be08869 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 22 Jul 2024 15:33:38 +0200 Subject: [PATCH 07/13] Make logs less spammy Co-authored-by: Markus Pettersson --- test/test-manager/src/logging.rs | 2 ++ test/test-manager/src/mullvad_daemon.rs | 2 +- test/test-manager/src/network_monitor.rs | 6 +++--- test/test-manager/src/vm/network/linux.rs | 2 +- test/test-manager/src/vm/provision.rs | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test-manager/src/logging.rs b/test/test-manager/src/logging.rs index 3d1aa1315677..2bc5ea90d15b 100644 --- a/test/test-manager/src/logging.rs +++ b/test/test-manager/src/logging.rs @@ -30,6 +30,8 @@ impl Logger { logger.filter_module("tower", log::LevelFilter::Info); logger.filter_module("hyper", log::LevelFilter::Info); logger.filter_module("rustls", log::LevelFilter::Info); + logger.filter_module("tarpc", log::LevelFilter::Warn); + logger.filter_module("mio_serial", log::LevelFilter::Warn); logger.filter_level(log::LevelFilter::Info); logger.parse_env(env_logger::DEFAULT_FILTER_ENV); diff --git a/test/test-manager/src/mullvad_daemon.rs b/test/test-manager/src/mullvad_daemon.rs index 115a09e049cc..251ed3d1fe99 100644 --- a/test/test-manager/src/mullvad_daemon.rs +++ b/test/test-manager/src/mullvad_daemon.rs @@ -69,7 +69,7 @@ impl RpcClientProvider { // FIXME: Ugly workaround to ensure that we don't receive stuff from a // previous RPC session. tokio::time::sleep(std::time::Duration::from_millis(500)).await; - log::debug!("Mullvad daemon: connecting"); + log::trace!("Mullvad daemon: connecting"); let channel = tonic::transport::Endpoint::from_static("serial://placeholder") .timeout(GRPC_REQUEST_TIMEOUT) .connect_with_connector(self.service.clone()) diff --git a/test/test-manager/src/network_monitor.rs b/test/test-manager/src/network_monitor.rs index ddd35edc7e9e..1a5ef5e4632d 100644 --- a/test/test-manager/src/network_monitor.rs +++ b/test/test-manager/src/network_monitor.rs @@ -59,7 +59,7 @@ impl PacketCodec for Codec { EtherTypes::Ipv4 => Self::parse_ipv4(frame.payload()), EtherTypes::Ipv6 => Self::parse_ipv6(frame.payload()), ethertype => { - log::debug!("Ignoring unknown ethertype: {ethertype}"); + log::trace!("Ignoring unknown ethertype: {ethertype}"); None } } @@ -288,11 +288,11 @@ async fn start_packet_monitor_for_interface( Some(Ok(packet))=> { if let Some(packet) = packet { if !filter_fn(&packet) { - log::debug!("{interface} \"{packet:?}\" does not match closure conditions"); + log::trace!("{interface} \"{packet:?}\" does not match closure conditions"); monitor_result.discarded_packets = monitor_result.discarded_packets.saturating_add(1); } else { - log::debug!("{interface} \"{packet:?}\" matches closure conditions"); + log::trace!("{interface} \"{packet:?}\" matches closure conditions"); let should_continue = should_continue_fn(&packet); diff --git a/test/test-manager/src/vm/network/linux.rs b/test/test-manager/src/vm/network/linux.rs index 1a4e994a52e0..12095138f65d 100644 --- a/test/test-manager/src/vm/network/linux.rs +++ b/test/test-manager/src/vm/network/linux.rs @@ -106,7 +106,7 @@ pub async fn setup_test_network() -> Result { let test_subnet = TEST_SUBNET.to_string(); - log::info!("Create bridge network: dev {BRIDGE_NAME}, net {test_subnet}"); + log::debug!("Create bridge network: dev {BRIDGE_NAME}, net {test_subnet}"); run_ip_cmd(["link", "add", BRIDGE_NAME, "type", "bridge"]).await?; run_ip_cmd(["addr", "add", "dev", BRIDGE_NAME, &test_subnet]).await?; diff --git a/test/test-manager/src/vm/provision.rs b/test/test-manager/src/vm/provision.rs index 42a12eb4261b..9a0153abd1b9 100644 --- a/test/test-manager/src/vm/provision.rs +++ b/test/test-manager/src/vm/provision.rs @@ -18,7 +18,7 @@ pub async fn provision( ) -> Result { match config.provisioner { Provisioner::Ssh => { - log::info!("SSH provisioning"); + log::debug!("SSH provisioning"); let (user, password) = config.get_ssh_options().context("missing SSH config")?; ssh( From 8b758874f9827d986e1273a5633f2cb917701c00 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 5 Aug 2024 12:03:49 +0200 Subject: [PATCH 08/13] Improve error messages --- test/test-manager/src/tests/account.rs | 11 +++++++---- test/test-manager/src/tests/ui.rs | 2 +- test/test-manager/src/vm/provision.rs | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/test/test-manager/src/tests/account.rs b/test/test-manager/src/tests/account.rs index 2cc7c5a95ecd..c67e1bed190c 100644 --- a/test/test-manager/src/tests/account.rs +++ b/test/test-manager/src/tests/account.rs @@ -97,10 +97,13 @@ pub async fn test_too_many_devices( log::info!("Log in with too many devices"); let login_result = login_with_retries(&mut mullvad_client).await; - assert!(matches!( - login_result, - Err(mullvad_management_interface::Error::TooManyDevices) - )); + assert!( + matches!( + login_result, + Err(mullvad_management_interface::Error::TooManyDevices) + ), + "Expected too many devices error, got {login_result:?}" + ); // Run UI test let ui_result = ui::run_test_env( diff --git a/test/test-manager/src/tests/ui.rs b/test/test-manager/src/tests/ui.rs index f4b9af04f3ee..f2b266a4e653 100644 --- a/test/test-manager/src/tests/ui.rs +++ b/test/test-manager/src/tests/ui.rs @@ -82,7 +82,7 @@ pub async fn run_test_env< let stdout = std::str::from_utf8(&result.stdout).unwrap_or("invalid utf8"); let stderr = std::str::from_utf8(&result.stderr).unwrap_or("invalid utf8"); - log::debug!("UI test failed:\n\nstdout:\n\n{stdout}\n\n{stderr}\n"); + log::error!("UI test failed:\n\nstdout:\n\n{stdout}\n\n{stderr}\n"); } Ok(result) diff --git a/test/test-manager/src/vm/provision.rs b/test/test-manager/src/vm/provision.rs index 9a0153abd1b9..0b7b88e766fb 100644 --- a/test/test-manager/src/vm/provision.rs +++ b/test/test-manager/src/vm/provision.rs @@ -218,6 +218,7 @@ fn ssh_send_file( fn ssh_exec(session: &Session, command: &str) -> Result { let mut channel = session.channel_session()?; channel.exec(command)?; + let mut stderr_handle = channel.stderr(); let mut output = String::new(); channel.read_to_string(&mut output)?; channel.send_eof()?; @@ -228,7 +229,9 @@ fn ssh_exec(session: &Session, command: &str) -> Result { .exit_status() .context("Failed to obtain exit status")?; if exit_status != 0 { - log::error!("command failed: {command}\n{output}"); + let mut stderr = String::new(); + stderr_handle.read_to_string(&mut stderr).unwrap(); + log::error!("Command failed: command: {command}\n\noutput:\n{output}\n\nstderr: {stderr}"); bail!("command failed: {exit_status}"); } From f29fe95ff339ab4e599ab39e3fa182a257c331ad Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 5 Aug 2024 12:03:49 +0200 Subject: [PATCH 09/13] Capture errors in `systemclt` cmd --- test/test-runner/src/sys.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/test/test-runner/src/sys.rs b/test/test-runner/src/sys.rs index a53396d08fda..e1060124d7f4 100644 --- a/test/test-runner/src/sys.rs +++ b/test/test-runner/src/sys.rs @@ -434,7 +434,7 @@ impl EnvVar { #[cfg(target_os = "linux")] pub async fn set_daemon_environment(env: HashMap) -> Result<(), test_rpc::Error> { - use std::fmt::Write; + use std::{fmt::Write, ops::Not}; let mut override_content = String::new(); override_content.push_str("[Service]\n"); @@ -459,17 +459,31 @@ pub async fn set_daemon_environment(env: HashMap) -> Result<(), .await .map_err(|e| test_rpc::Error::Service(e.to_string()))?; - tokio::process::Command::new("systemctl") + if tokio::process::Command::new("systemctl") .args(["daemon-reload"]) .status() .await - .map_err(|e| test_rpc::Error::Service(e.to_string()))?; + .map_err(|e| test_rpc::Error::Io(e.to_string()))? + .success() + .not() + { + return Err(test_rpc::Error::Service( + "Daemon service could not be reloaded".to_owned(), + )); + }; - tokio::process::Command::new("systemctl") + if tokio::process::Command::new("systemctl") .args(["restart", "mullvad-daemon"]) .status() .await - .map_err(|e| test_rpc::Error::Service(e.to_string()))?; + .map_err(|e| test_rpc::Error::Io(e.to_string()))? + .success() + .not() + { + return Err(test_rpc::Error::Service( + "Daemon service could not be restarted".to_owned(), + )); + }; wait_for_service_state(ServiceState::Running).await?; Ok(()) From 27157eaa90d4b36fd0a9da53339c428c2d50a53b Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 5 Aug 2024 12:03:49 +0200 Subject: [PATCH 10/13] Fix `test_connected_state` failing on obfuscation != `None` --- test/test-manager/src/tests/tunnel_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-manager/src/tests/tunnel_state.rs b/test/test-manager/src/tests/tunnel_state.rs index e8ad102419e9..031a669db49b 100644 --- a/test/test-manager/src/tests/tunnel_state.rs +++ b/test/test-manager/src/tests/tunnel_state.rs @@ -366,7 +366,7 @@ pub async fn test_connected_state( tunnel_type: TunnelType::Wireguard, quantum_resistant: _, proxy: None, - obfuscation: None, + obfuscation: _, entry_endpoint: None, tunnel_interface: _, daita: _, From 0d9aac8123f26f3acd010e0733626191472ff8b9 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Mon, 5 Aug 2024 13:49:16 +0200 Subject: [PATCH 11/13] Restructure test scripts Split functionality `ci-runtest.sh` into multiple scripts. `test-by-version.sh` can be used to test against any version of the app available on the build servers. `test-utils.sh` contains shared logic. Rename `PACKAGES_DIR` env `PACKAGE_DIR`, it's more consistent with the new CLI flag. --- .github/workflows/desktop-e2e.yml | 6 +- test/README.md | 10 +- test/ci-runtests.sh | 246 ---------------------- test/{ => scripts}/Dockerfile | 0 test/scripts/build-runner-image.sh | 2 +- test/{ => scripts}/build-runner.sh | 5 +- test/scripts/ci-runtests.sh | 67 ++++++ test/{ => scripts}/container-run.sh | 14 +- test/scripts/test-utils.sh | 312 ++++++++++++++++++++++++++++ test/test-by-version.sh | 69 ++++++ test/test-manager/src/main.rs | 8 +- test/test-manager/src/package.rs | 14 +- 12 files changed, 478 insertions(+), 275 deletions(-) delete mode 100755 test/ci-runtests.sh rename test/{ => scripts}/Dockerfile (100%) rename test/{ => scripts}/build-runner.sh (87%) create mode 100755 test/scripts/ci-runtests.sh rename test/{ => scripts}/container-run.sh (67%) create mode 100755 test/scripts/test-utils.sh create mode 100755 test/test-by-version.sh diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index 03e2959be11f..23a612f9acbb 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -151,7 +151,7 @@ jobs: shell: bash -ieo pipefail {0} run: | git fetch --tags --force - ./test/ci-runtests.sh ${{ matrix.os }} + ./test/scripts/ci-runtests.sh ${{ matrix.os }} - uses: actions/upload-artifact@v3 if: '!cancelled()' with: @@ -226,7 +226,7 @@ jobs: shell: bash -ieo pipefail {0} run: | git fetch --tags --force - ./test/ci-runtests.sh ${{ matrix.os }} + ./test/scripts/ci-runtests.sh ${{ matrix.os }} - uses: actions/upload-artifact@v3 if: '!cancelled()' with: @@ -297,7 +297,7 @@ jobs: shell: bash -ieo pipefail {0} run: | git fetch --tags --force - ./test/ci-runtests.sh ${{ matrix.os }} + ./test/scripts/ci-runtests.sh ${{ matrix.os }} - uses: actions/upload-artifact@v3 if: '!cancelled()' with: diff --git a/test/README.md b/test/README.md index b490166ef523..c8ff94c4257e 100644 --- a/test/README.md +++ b/test/README.md @@ -66,27 +66,27 @@ Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` ( For example, building `test-runner` for Windows would look like this: ``` bash -./container-run.sh ./build-runner.sh windows +./scripts/container-run.sh ./scripts/build-runner.sh windows ``` ## Linux Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#Prerequisities) for more details. ``` bash -./container-run.sh ./build-runner.sh linux +./scripts/container-run.sh ./scripts/build-runner.sh linux ``` ## macOS ``` bash -./build-runner.sh macos +./scripts/build-runner.sh macos ``` ## Windows The `test-runner` binary for Windows may be cross-compiled from a Linux host. ``` bash -./container-run.sh ./build-runner.sh windows +./scripts/container-run.sh ./scripts/build-runner.sh windows ``` # Building base images @@ -145,7 +145,7 @@ cargo run --bin test-manager run-tests macos-ventura \ --app-package-to-upgrade-from 2023.2 ``` -## Note on `ci-runtests.sh` +## Note on `scripts/ci-runtests.sh` Account tokens are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM. diff --git a/test/ci-runtests.sh b/test/ci-runtests.sh deleted file mode 100755 index 0346044e7bdf..000000000000 --- a/test/ci-runtests.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -APP_DIR="$SCRIPT_DIR/../" -cd "$SCRIPT_DIR" - -BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases" -BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds" - -if [[ ("$(uname -s)" == "Darwin") ]]; then - export PACKAGES_DIR=$HOME/Library/Caches/mullvad-test/packages -elif [[ ("$(uname -s)" == "Linux") ]]; then - export PACKAGES_DIR=$HOME/.cache/mullvad-test/packages -else - echo "Unsupported OS" 1>&2 - exit 1 -fi - -if [[ "$#" -lt 1 ]]; then - echo "usage: $0 TEST_OS" 1>&2 - exit 1 -fi - -TEST_OS=$1 - -# Infer stable version from GitHub repo -RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]') -OLD_APP_VERSION=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES") - -NEW_APP_VERSION=$(cargo run -q --manifest-path="$APP_DIR/Cargo.toml" --bin mullvad-version) -commit=$(git rev-parse HEAD^\{commit\}) -commit=${commit:0:6} - -TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "") - -if [[ -n "$TAG" && ${NEW_APP_VERSION} =~ -dev- ]]; then - NEW_APP_VERSION+="+${TAG}" -fi - -echo "**********************************" -echo "* Version to upgrade from: $OLD_APP_VERSION" -echo "* Version to test: $NEW_APP_VERSION" -echo "**********************************" - - -if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then - echo "'ACCOUNT_TOKENS' must be specified" 1>&2 - exit 1 -fi -if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then - echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2 - exit 1 -fi - -mkdir -p "$SCRIPT_DIR/.ci-logs" -echo "$NEW_APP_VERSION" > "$SCRIPT_DIR/.ci-logs/last-version.log" - -function nice_time { - SECONDS=0 - if "$@"; then - result=0 - else - result=$? - fi - s=$SECONDS - echo "\"$*\" completed in $((s/60))m:$((s%60))s" - return $result -} - -# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups -# if that is the case. -function is_dev_version { - local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$" - if [[ "$1" =~ $pattern ]]; then - return 0 - fi - return 1 -} - -function get_app_filename { - local version=$1 - local os=$2 - if is_dev_version "$version"; then - # only save 6 chars of the hash - local commit="${BASH_REMATCH[3]}" - version="${BASH_REMATCH[1]}${commit}" - # If the dev-version includes a tag, we need to append it to the app filename - if [[ -n ${BASH_REMATCH[4]} ]]; then - version="${version}${BASH_REMATCH[4]}" - fi - fi - case $os in - debian*|ubuntu*) - echo "MullvadVPN-${version}_amd64.deb" - ;; - fedora*) - echo "MullvadVPN-${version}_x86_64.rpm" - ;; - windows*) - echo "MullvadVPN-${version}.exe" - ;; - macos*) - echo "MullvadVPN-${version}.pkg" - ;; - *) - echo "Unsupported target: $os" 1>&2 - return 1 - ;; - esac -} - -function download_app_package { - local version=$1 - local os=$2 - local package_repo="" - - if is_dev_version "$version"; then - package_repo="${BUILD_DEV_REPOSITORY}" - else - package_repo="${BUILD_RELEASE_REPOSITORY}" - fi - - local filename - filename=$(get_app_filename "$version" "$os") - local url="${package_repo}/$version/$filename" - - mkdir -p "$PACKAGES_DIR" - if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then - echo "Downloading build for $version ($os) from $url" - curl -sf -o "$PACKAGES_DIR/$filename" "$url" - else - echo "Found build for $version ($os)" - fi -} - -function get_e2e_filename { - local version=$1 - local os=$2 - if is_dev_version "$version"; then - # only save 6 chars of the hash - local commit="${BASH_REMATCH[3]}" - version="${BASH_REMATCH[1]}${commit}" - fi - case $os in - debian*|ubuntu*|fedora*) - echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu" - ;; - windows*) - echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe" - ;; - macos*) - echo "app-e2e-tests-${version}-aarch64-apple-darwin" - ;; - *) - echo "Unsupported target: $os" 1>&2 - return 1 - ;; - esac -} - -function download_e2e_executable { - local version=$1 - local os=$2 - local package_repo="" - - if is_dev_version "$version"; then - package_repo="${BUILD_DEV_REPOSITORY}" - else - package_repo="${BUILD_RELEASE_REPOSITORY}" - fi - - local filename - filename=$(get_e2e_filename "$version" "$os") - local url="${package_repo}/$version/additional-files/$filename" - - mkdir -p "$PACKAGES_DIR" - if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then - echo "Downloading e2e executable for $version ($os) from $url" - curl -sf -o "$PACKAGES_DIR/$filename" "$url" - else - echo "Found e2e executable for $version ($os)" - fi -} - -function run_tests_for_os { - local os=$1 - - local prev_filename - prev_filename=$(get_app_filename "$OLD_APP_VERSION" "$os") - local cur_filename - cur_filename=$(get_app_filename "$NEW_APP_VERSION" "$os") - - rm -f "$SCRIPT_DIR/.ci-logs/${os}_report" - - RUST_LOG=debug cargo run --bin test-manager \ - run-tests \ - --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \ - --app-package "${cur_filename}" \ - --app-package-to-upgrade-from "${prev_filename}" \ - --package-folder "$PACKAGES_DIR" \ - --test-report "$SCRIPT_DIR/.ci-logs/${os}_report" \ - --vm "$os" 2>&1 | sed "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g" -} - -echo "**********************************" -echo "* Downloading app packages" -echo "**********************************" - -find "$PACKAGES_DIR" -type f -mtime +5 -delete || true - -mkdir -p "$PACKAGES_DIR" -nice_time download_app_package "$OLD_APP_VERSION" "$TEST_OS" -nice_time download_app_package "$NEW_APP_VERSION" "$TEST_OS" -nice_time download_e2e_executable "$NEW_APP_VERSION" "$TEST_OS" - -echo "**********************************" -echo "* Building test runner" -echo "**********************************" - -function build_test_runner { - if [[ "${TEST_OS}" =~ "debian"|"ubuntu"|"fedora" ]]; then - ./container-run.sh ./build-runner.sh linux - elif [[ "${TEST_OS}" =~ "windows" ]]; then - ./container-run.sh ./build-runner.sh windows - elif [[ "${TEST_OS}" =~ "macos" ]]; then - ./build-runner.sh macos - fi -} - -nice_time build_test_runner - -echo "**********************************" -echo "* Building test manager" -echo "**********************************" - -cargo build -p test-manager - -echo "**********************************" -echo "* Running tests" -echo "**********************************" - -mkdir -p "$SCRIPT_DIR/.ci-logs/os/" -set -o pipefail -ACCOUNT_TOKEN=${tokens[0]} nice_time run_tests_for_os "${TEST_OS}" diff --git a/test/Dockerfile b/test/scripts/Dockerfile similarity index 100% rename from test/Dockerfile rename to test/scripts/Dockerfile diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build-runner-image.sh index 4d8b39267d68..4aec7b04397c 100755 --- a/test/scripts/build-runner-image.sh +++ b/test/scripts/build-runner-image.sh @@ -34,7 +34,7 @@ case $TARGET in -i "${TEST_RUNNER_IMAGE_PATH}" \ "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \ "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \ - "${PACKAGES_DIR}/"*.exe \ + "${PACKAGE_DIR}/"*.exe \ "${SCRIPT_DIR}/../openvpn.ca.crt" \ "::" mdir -i "${TEST_RUNNER_IMAGE_PATH}" diff --git a/test/build-runner.sh b/test/scripts/build-runner.sh similarity index 87% rename from test/build-runner.sh rename to test/scripts/build-runner.sh index ef25d001c35b..ffdb2b861a26 100755 --- a/test/build-runner.sh +++ b/test/scripts/build-runner.sh @@ -3,9 +3,10 @@ set -eu SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_DIR="$SCRIPT_DIR/.." +REPO_DIR="$SCRIPT_DIR/../.." cd "$SCRIPT_DIR" +# shellcheck disable=SC1091 source "$REPO_DIR/scripts/utils/log" case ${1-:""} in @@ -34,5 +35,5 @@ cargo build \ # Only build runner image for Windows if [[ $TARGET == x86_64-pc-windows-gnu ]]; then - TARGET="$TARGET" ./scripts/build-runner-image.sh + TARGET="$TARGET" ./build-runner-image.sh fi diff --git a/test/scripts/ci-runtests.sh b/test/scripts/ci-runtests.sh new file mode 100755 index 000000000000..3e64894e744f --- /dev/null +++ b/test/scripts/ci-runtests.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -eu + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" +TEST_DIR="$SCRIPT_DIR/.." + +if [[ "$#" -lt 1 ]]; then + echo "usage: $0 TEST_OS" 1>&2 + exit 1 +fi + +TEST_OS=$1 + +# shellcheck source=test/scripts/test-utils.sh +source "test-utils.sh" + +echo "**********************************" +echo "* Version to upgrade from: $LATEST_STABLE_RELEASE" +echo "* Version to test: $CURRENT_VERSION" +echo "**********************************" + + +if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then + echo "'ACCOUNT_TOKENS' must be specified" 1>&2 + exit 1 +fi +if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then + echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2 + exit 1 +fi +CI_LOGS_DIR="$TEST_DIR/.ci-logs" +mkdir -p "$CI_LOGS_DIR" +echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log" + + +echo "**********************************" +echo "* Downloading app packages" +echo "**********************************" + + +nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS" +nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS" +nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS" + +echo "**********************************" +echo "* Building test manager" +echo "**********************************" + +cargo build -p test-manager + +echo "**********************************" +echo "* Running tests" +echo "**********************************" + +mkdir -p "$CI_LOGS_DIR/os/" +export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report" +rm -f "$TEST_REPORT" + +set -o pipefail + +APP_PACKAGE=$(get_app_filename "$CURRENT_VERSION" "$TEST_OS") +export APP_PACKAGE +APP_PACKAGE_TO_UPGRADE_FROM=$(get_app_filename "$LATEST_STABLE_RELEASE" "$TEST_OS") +export APP_PACKAGE_TO_UPGRADE_FROM +ACCOUNT_TOKEN=${tokens[0]} RUST_LOG=debug nice_time run_tests_for_os "${TEST_OS}" diff --git a/test/container-run.sh b/test/scripts/container-run.sh similarity index 67% rename from test/container-run.sh rename to test/scripts/container-run.sh index e4f24a29d930..4f87655123f7 100755 --- a/test/container-run.sh +++ b/test/scripts/container-run.sh @@ -4,16 +4,17 @@ set -eu CARGO_REGISTRY_VOLUME_NAME=${CARGO_REGISTRY_VOLUME_NAME:-"cargo-registry"} CONTAINER_RUNNER=${CONTAINER_RUNNER:-"podman"} -PACKAGES_DIR=${PACKAGES_DIR:-"$HOME/.cache/mullvad-test/packages"} +PACKAGE_DIR=${PACKAGE_DIR:-"$HOME/.cache/mullvad-test/packages"} -if [ ! -d "$PACKAGES_DIR" ]; then - echo "$PACKAGES_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script." +if [ ! -d "$PACKAGE_DIR" ]; then + echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script." fi SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_DIR="$SCRIPT_DIR/.." +REPO_DIR="$SCRIPT_DIR/../.." cd "$SCRIPT_DIR" +# shellcheck disable=SC1091 source "$REPO_DIR/scripts/utils/log" if [[ "$(uname -s)" != "Linux" ]]; then @@ -24,13 +25,12 @@ fi container_image=$(cat "$REPO_DIR/building/linux-container-image.txt") podman build -t mullvadvpn-app-tests --build-arg IMAGE="${container_image}" . -set -x exec "$CONTAINER_RUNNER" run --rm -it \ -v "${CARGO_REGISTRY_VOLUME_NAME}":/root/.cargo/registry:Z \ -v "${REPO_DIR}":/build:z \ -w "/build/test" \ -e CARGO_TARGET_DIR=/build/test/target \ - -v "${PACKAGES_DIR}":/packages:Z \ - -e PACKAGES_DIR=/packages \ + -v "${PACKAGE_DIR}":/packages:Z \ + -e PACKAGE_DIR=/packages \ mullvadvpn-app-tests \ /bin/bash -c "$*" diff --git a/test/scripts/test-utils.sh b/test/scripts/test-utils.sh new file mode 100755 index 000000000000..c73c79e52a55 --- /dev/null +++ b/test/scripts/test-utils.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash + +set -eu + +# Returns the directory of the test-utils.sh script +function get_test_utls_dir { + local script_path="${BASH_SOURCE[0]}" + local script_dir + if [[ -n "$script_path" ]]; then + script_dir="$(cd "$(dirname "$script_path")" > /dev/null && pwd)" + else + script_dir="$(cd "$(dirname "$0")" > /dev/null && pwd)" + fi + echo "$script_dir" +} + +export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases" +export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds" + +# Infer stable version from GitHub repo +RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]') +LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES") + +function get_current_version { + local app_dir + app_dir="$(get_test_utls_dir)/../.." + cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version +} + +CURRENT_VERSION=$(get_current_version) +commit=$(git rev-parse HEAD^\{commit\}) +commit=${commit:0:6} + +TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "") + +if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then + CURRENT_VERSION+="+${TAG}" +fi + +export CURRENT_VERSION +export LATEST_STABLE_RELEASE + +function print_available_releases { + for release in $(jq -r '.[].tag_name'<<<"$RELEASES"); do + echo "$release" + done +} + +function get_package_dir { + local package_dir + if [[ -n "${PACKAGE_DIR+x}" ]]; then + # Resolve the package dir to an absolute path since cargo must be invoked from the test directory + package_dir=$(cd "$PACKAGE_DIR" > /dev/null && pwd) + elif [[ ("$(uname -s)" == "Darwin") ]]; then + package_dir="$HOME/Library/Caches/mullvad-test/packages" + elif [[ ("$(uname -s)" == "Linux") ]]; then + package_dir="$HOME/.cache/mullvad-test/packages" + else + echo "Unsupported OS" 1>&2 + exit 1 + fi + + mkdir -p "$package_dir" || exit 1 + # Clean up old packages + find "$package_dir" -type f -mtime +5 -delete || true + + echo "$package_dir" + return 0 +} + +function nice_time { + SECONDS=0 + if "$@"; then + result=0 + else + result=$? + fi + s=$SECONDS + echo "\"$*\" completed in $((s/60))m:$((s%60))s" + return $result +} + +# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups +# if that is the case. +function is_dev_version { + local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$" + if [[ "$1" =~ $pattern ]]; then + return 0 + fi + return 1 +} + +function get_app_filename { + local version=$1 + local os=$2 + if is_dev_version "$version"; then + # only save 6 chars of the hash + local commit="${BASH_REMATCH[3]}" + version="${BASH_REMATCH[1]}${commit}" + # If the dev-version includes a tag, we need to append it to the app filename + if [[ -n ${BASH_REMATCH[4]} ]]; then + version="${version}${BASH_REMATCH[4]}" + fi + fi + case $os in + debian*|ubuntu*) + echo "MullvadVPN-${version}_amd64.deb" + ;; + fedora*) + echo "MullvadVPN-${version}_x86_64.rpm" + ;; + windows*) + echo "MullvadVPN-${version}.exe" + ;; + macos*) + echo "MullvadVPN-${version}.pkg" + ;; + *) + echo "Unsupported target: $os" 1>&2 + return 1 + ;; + esac +} + +function download_app_package { + local version=$1 + local os=$2 + local package_repo="" + + if is_dev_version "$version"; then + package_repo="${BUILD_DEV_REPOSITORY}" + else + package_repo="${BUILD_RELEASE_REPOSITORY}" + fi + + local filename + filename=$(get_app_filename "$version" "$os") + local url="${package_repo}/$version/$filename" + + local package_dir + package_dir=$(get_package_dir) + if [[ ! -f "$package_dir/$filename" ]]; then + echo "Downloading build for $version ($os) from $url" + if ! curl -sf -o "$package_dir/$filename" "$url"; then + echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2 + exit 1 + fi + else + echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download" + fi +} + +function get_e2e_filename { + local version=$1 + local os=$2 + if is_dev_version "$version"; then + # only save 6 chars of the hash + local commit="${BASH_REMATCH[3]}" + version="${BASH_REMATCH[1]}${commit}" + fi + case $os in + debian*|ubuntu*|fedora*) + echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu" + ;; + windows*) + echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe" + ;; + macos*) + echo "app-e2e-tests-${version}-aarch64-apple-darwin" + ;; + *) + echo "Unsupported target: $os" 1>&2 + return 1 + ;; + esac +} + +function download_e2e_executable { + local version=${1:?Error: version not set} + local os=${2:?Error: os not set} + local package_repo + + if is_dev_version "$version"; then + package_repo="${BUILD_DEV_REPOSITORY}" + else + package_repo="${BUILD_RELEASE_REPOSITORY}" + fi + + local filename + filename=$(get_e2e_filename "$version" "$os") + local url="${package_repo}/$version/additional-files/$filename" + + local package_dir + package_dir=$(get_package_dir) + if [[ ! -f "$package_dir/$filename" ]]; then + echo "Downloading e2e executable for $version ($os) from $url" + if ! curl -sf -o "$package_dir/$filename" "$url"; then + echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2 + exit 1 + fi + else + echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download" + fi +} + +function build_test_runner { + local script_dir + script_dir=$(get_test_utls_dir) + local test_os=${1:?Error: test os not set} + if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then + "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1 + elif [[ "${test_os}" =~ "windows" ]]; then + "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1 + elif [[ "${test_os}" =~ "macos" ]]; then + "$script_dir"/build-runner.sh macos || exit 1 + fi +} + +function run_tests_for_os { + local vm=$1 + + if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then + echo "'ACCOUNT_TOKEN' must be specified" 1>&2 + exit 1 + fi + + echo "**********************************" + echo "* Building test runner" + echo "**********************************" + + nice_time build_test_runner "$vm" + + + echo "**********************************" + echo "* Running tests" + echo "**********************************" + + local upgrade_package_arg + if [[ -z "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then + echo "'APP_PACKAGE_TO_UPGRADE_FROM' env not set, not testing upgrades" + upgrade_package_arg=() + else + upgrade_package_arg=(--app-package-to-upgrade-from "${APP_PACKAGE_TO_UPGRADE_FROM}") + fi + + if [[ -z "${TEST_REPORT+x}" ]]; then + echo "'TEST_REPORT' env not set, not saving test report" + test_report_arg=() + else + test_report_arg=(--test-report "${TEST_REPORT}") + fi + + local package_dir + package_dir=$(get_package_dir) + local test_dir + test_dir=$(get_test_utls_dir)/.. + pushd "$test_dir" + if ! RUST_LOG_STYLE=always cargo run --bin test-manager \ + run-tests \ + --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \ + --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \ + "${upgrade_package_arg[@]}" \ + "${test_report_arg[@]}" \ + --package-dir "${package_dir}" \ + --vm "$vm" \ + "${TEST_FILTERS:-}" \ + 2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then + echo "Test run failed" + exit 1 + fi + popd +} + +# Build the current version of the app and move the package to the package folder +# Currently unused, but may be useful in the future +function build_current_version { + local app_dir + app_dir="$(get_test_utls_dir)/../.." + local app_filename + # TODO: TEST_OS must be set to local OS manually, should be set automatically + app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") + local package_dir + package_dir=$(get_package_dir) + local app_package="$package_dir"/"$app_filename" + + local gui_test_filename + gui_test_filename=$(get_e2e_filename "$CURRENT_VERSION" "$TEST_OS") + local gui_test_bin="$package_dir"/"$gui_test_filename" + + if [ ! -f "$app_package" ]; then + pushd "$app_dir" + if [[ $(git diff --quiet) ]]; then + echo "WARNING: the app repository contains uncommitted changes, this script will only rebuild the app package when the git hash changes" + fi + ./build.sh + popd + echo "Moving '$(realpath "$app_dir/dist/$app_filename")' to '$(realpath "$app_package")'" + mv -n "$app_dir"/dist/"$app_filename" "$app_package" + else + echo "App package for current version already exists at $app_package, skipping build" + fi + + if [ ! -f "$gui_test_bin" ]; then + pushd "$app_dir"/gui + npm run build-test-executable + popd + echo "Moving '$(realpath "$app_dir/dist/$gui_test_filename")' to '$(realpath "$gui_test_bin")'" + mv -n "$app_dir"/dist/"$gui_test_filename" "$gui_test_bin" + else + echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build" + fi +} \ No newline at end of file diff --git a/test/test-by-version.sh b/test/test-by-version.sh new file mode 100755 index 000000000000..a86dac8cedc5 --- /dev/null +++ b/test/test-by-version.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -eu + +usage() { + echo "This script downloads and tests the given app version from the build repositories." + echo + echo "Required environment variables:" + echo " - ACCOUNT_TOKEN: Valid MullvadVPN account token" + echo " - TEST_OS: Name of the VM configuration to use. List available configurations with 'cargo run --bin test-manager list'" + echo "Optional environment variables:" + echo " - APP_VERSION: The version of the app to test (defaults to the latest stable release)" + echo " - APP_PACKAGE_TO_UPGRADE_FROM: The package version to upgrade from (defaults to none)" + echo " - TEST_FILTERS: specifies which tests to run (defaults to all)" + echo " - TEST_REPORT : path to save the test results in a structured format" +} + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +# shellcheck source=test/scripts/test-utils.sh +source "scripts/test-utils.sh" + +if [[ ( "$*" == "--help") || "$*" == "-h" ]]; then + usage + exit 0 +fi + +if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then + echo "'ACCOUNT_TOKEN' must be specified" 1>&2 + echo + usage + exit 1 +fi + +if [[ -z "${TEST_OS+x}" ]]; then + echo "'TEST_OS' must be specified" 1>&2 + echo + usage + exit 1 +fi + +if [[ -z "${APP_VERSION+x}" ]]; then + echo "'APP_VERSION' not set, using latest build from the list of GitHub releases:" + print_available_releases + echo "For a full list of available releases you can choose from, see the stable build repository: $BUILD_RELEASE_REPOSITORY" + echo "and the dev build repository: $BUILD_DEV_REPOSITORY" + APP_VERSION=$LATEST_STABLE_RELEASE +fi + +echo "**********************************" +echo "* Version to test: $APP_VERSION" +echo "**********************************" + +echo "**********************************" +echo "* Downloading app packages" +echo "**********************************" + +download_app_package "$APP_VERSION" "$TEST_OS" +download_e2e_executable "$APP_VERSION" "$TEST_OS" + +if [[ -n "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then + download_app_package "$APP_PACKAGE_TO_UPGRADE_FROM" "$TEST_OS" +fi + +set -o pipefail +APP_PACKAGE=$(get_app_filename "$APP_VERSION" "$TEST_OS") +export APP_PACKAGE +run_tests_for_os "${TEST_OS}" diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs index bd9954bc1e63..daad04983bdc 100644 --- a/test/test-manager/src/main.rs +++ b/test/test-manager/src/main.rs @@ -82,7 +82,7 @@ enum Commands { /// App package to test. Can be a path to the package, just the package file name, git hash /// or tag. If the direct path is not given, the package is assumed to be in the directory - /// specified by the `--package-folder` argument. + /// specified by the `--package-dir` argument. /// /// # Note /// @@ -108,7 +108,7 @@ enum Commands { /// Folder to search for packages. Defaults to current directory. #[arg(long, value_name = "DIR")] - package_folder: Option, + package_dir: Option, /// Only run tests matching substrings test_filters: Vec, @@ -227,7 +227,7 @@ async fn main() -> Result<()> { app_package, app_package_to_upgrade_from, gui_package, - package_folder, + package_dir, test_filters, verbose, test_report, @@ -266,7 +266,7 @@ async fn main() -> Result<()> { app_package, app_package_to_upgrade_from, gui_package, - package_folder, + package_dir, ) .context("Could not find the specified app packages")?; diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs index b6147d7dd8de..34fb30419a7f 100644 --- a/test/test-manager/src/package.rs +++ b/test/test-manager/src/package.rs @@ -21,22 +21,22 @@ pub fn get_app_manifest( app_package: String, app_package_to_upgrade_from: Option, gui_package: Option, - package_folder: Option, + package_dir: Option, ) -> Result { let package_type = (config.os_type, config.package_type, config.architecture); - let app_package_path = find_app(&app_package, false, package_type, package_folder.as_ref())?; + let app_package_path = find_app(&app_package, false, package_type, package_dir.as_ref())?; log::info!("App package: {}", app_package_path.display()); let app_package_to_upgrade_from_path = app_package_to_upgrade_from - .map(|app| find_app(&app, false, package_type, package_folder.as_ref())) + .map(|app| find_app(&app, false, package_type, package_dir.as_ref())) .transpose()?; log::info!("App package to upgrade from: {app_package_to_upgrade_from_path:?}"); // Automatically try to find the UI e2e tests based on the app package // Search the specified package folder, or same folder as the app package if missing - let ui_e2e_package_folder = package_folder.unwrap_or( + let ui_e2e_package_dir = package_dir.unwrap_or( app_package_path .parent() .expect("Path to app package should have parent") @@ -51,7 +51,7 @@ pub fn get_app_manifest( }, true, package_type, - Some(&ui_e2e_package_folder), + Some(&ui_e2e_package_dir), ); // Don't allow the UI/e2e test binary to missing if it's flag was specified @@ -85,7 +85,7 @@ fn find_app( app: &str, e2e_bin: bool, package_type: (OsType, Option, Option), - package_folder: Option<&PathBuf>, + package_dir: Option<&PathBuf>, ) -> Result { // If it's a path, use that path let app_path = Path::new(app); @@ -98,7 +98,7 @@ fn find_app( app.make_ascii_lowercase(); let current_dir = std::env::current_dir().expect("Unable to get current directory"); - let packages_dir = package_folder.unwrap_or(¤t_dir); + let packages_dir = package_dir.unwrap_or(¤t_dir); std::fs::create_dir_all(packages_dir)?; let dir = std::fs::read_dir(packages_dir.clone()).context("Failed to list packages")?; From db134abfa31134d1fb71ce052039aff168b9bdbf Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Tue, 6 Aug 2024 14:48:49 +0200 Subject: [PATCH 12/13] Fix `tarpc: Connection broken` error --- test/test-manager/src/run_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-manager/src/run_tests.rs b/test/test-manager/src/run_tests.rs index 73c72ebc2ef5..4449a06a6f3b 100644 --- a/test/test-manager/src/run_tests.rs +++ b/test/test-manager/src/run_tests.rs @@ -169,6 +169,7 @@ pub async fn run( } // wait for cleanup + drop(client); drop(test_context); let _ = tokio::time::timeout(Duration::from_secs(5), completion_handle).await; From c29f619784da0ad10d43832c39ee7e15230d1878 Mon Sep 17 00:00:00 2001 From: Sebastian Holmin Date: Wed, 7 Aug 2024 11:55:21 +0200 Subject: [PATCH 13/13] Update README.md --- test/README.md | 120 ++++++++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 51 deletions(-) diff --git a/test/README.md b/test/README.md index c8ff94c4257e..6aaff56d2f1c 100644 --- a/test/README.md +++ b/test/README.md @@ -1,31 +1,34 @@ -# Project structure +# Mullvad VPN end to end test framework -## test-manager +## Project structure + +### test-manager The client part of the testing environment. This program runs on the host and connects over a virtual serial port to the `test-runner`. The tests themselves are defined in this package, using the interface provided by `test-runner`. -## test-runner +### test-runner The server part of the testing environment. This program runs in guest VMs and provides the `test-manager` with the building blocks (RPCs) needed to create tests. -## test-rpc +### test-rpc A support library for the other two packages. Defines an RPC interface, transports, shared types, etc. -# Prerequisities + +## Prerequisites For macOS, the host machine must be macOS. All other platforms assume that the host is Linux. -## All platforms +### All platforms * Get the latest stable Rust from https://rustup.rs/. -## macOS +### macOS Normally, you would use Tart here. It can be installed with Homebrew. You'll also need `wireguard-tools`, a protobuf compiler, and OpenSSL: @@ -34,7 +37,7 @@ Normally, you would use Tart here. It can be installed with Homebrew. You'll als brew install cirruslabs/cli/tart wireguard-tools pkg-config openssl protobuf ``` -### Wireshark +#### Wireshark Wireshark is also required. More specifically, you'll need `wireshark-chmodbpf`, which can be found in the Wireshark installer here: https://www.wireshark.org/download.html @@ -47,7 +50,7 @@ dseditgroup -o edit -a THISUSER -t user access_bpf This lets us monitor traffic on network interfaces without root access. -## Linux +### Linux For running tests on Linux and Windows guests, you will need these tools and libraries: @@ -58,7 +61,51 @@ dnf install git gcc protobuf-devel libpcap-devel qemu \ wireguard-tools ``` -# Building the test runner +## Setting up testing environment + +First you need to build the images for running tests on, see [`BUILD_OS_IMAGE.md`](./docs/BUILD_OS_IMAGE.md). The `test-manager` then needs to be configured to use the image. + +Here is an example of how to create a new OS configuration for Linux and macOS: + +### Linux + +```bash +# Create or edit configuration +# The image is assumed to contain a test runner service set up as described in ./docs/BUILD_OS_IMAGE.md +cargo run --bin test-manager set debian11 qemu ./os-images/debian11.qcow2 linux \ + --package-type deb --architecture x64 \ + --provisioner ssh --ssh-user test --ssh-password test + +# Try it out to see if it works - you should reach the VM's graphical desktop environment +cargo run --bin test-manager run-vm debian11 +``` + +### macOS + + +```bash +# Download some VM image +tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base + +# Create or edit configuration +# Use SSH to deploy the test runner since the image doesn't contain a runner +cargo run --bin test-manager set macos-ventura tart ventura-base macos \ + --architecture aarch64 \ + --provisioner ssh --ssh-user admin --ssh-password admin + +# Try it out to see if it works +cargo run -p test-manager run-vm macos-ventura +``` + +## Testing the app + +To automatically download and test a pre-built version of the app, use the `test-by-version.sh` script, see `test-by-version.sh --help` for instructions. + +To manually invoke `test-manager`, start by checking out the desired git version of this repo. Next, [build the app](../BuildInstructions.md) for the target platform then build the GUI test binary using `$(cd ../gui && npm run build-test-executable)`. The newly built packages will be located in the `../dist` folder by default. + +Next: build the `test-runner` + +### Building the test runner Building the `test-runner` binary is done with the `build-runner.sh` script. Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (Apple Silicon) for macOS. @@ -69,76 +116,48 @@ For example, building `test-runner` for Windows would look like this: ./scripts/container-run.sh ./scripts/build-runner.sh windows ``` -## Linux -Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#Prerequisities) for more details. +#### Linux +Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#prerequisites) for more details. ``` bash ./scripts/container-run.sh ./scripts/build-runner.sh linux ``` -## macOS +#### macOS ``` bash ./scripts/build-runner.sh macos ``` -## Windows +#### Windows The `test-runner` binary for Windows may be cross-compiled from a Linux host. ``` bash ./scripts/container-run.sh ./scripts/build-runner.sh windows ``` -# Building base images - -See [`BUILD_OS_IMAGE.md`](./docs/BUILD_OS_IMAGE.md) for how to build images for running tests on. - -# Running tests +### Running the tests -See `cargo run --bin test-manager` for details. +After configuring the VM image using `test-manager set` and building the required packages (see [previous step](#setting-up-testing-environment)), `test-manager run-tests` is used to launch the tests. See `cargo run --bin test-manager -- run-tests --help` for details. -## Linux +Here is an example of how to run all tests using the Linux/macOS VM we set up earlier: -Here is an example of how to create a new OS configuration and then run all tests: +#### Linux ```bash -# Create or edit configuration -# The image is assumed to contain a test runner service set up as described in ./docs/BUILD_OS_IMAGE.md -cargo run --bin test-manager set debian11 qemu ./os-images/debian11.qcow2 linux \ - --package-type deb --architecture x64 \ - --provisioner ssh --ssh-user test --ssh-password test - -# Try it out to see if it works - you should reach the VM's graphical desktop environment -cargo run --bin test-manager run-vm debian11 - # Run all tests -cargo run --bin test-manager run-tests debian11 \ +cargo run --bin test-manager run-tests --vm debian11 \ --display \ --account 0123456789 \ --app-package \ --app-package-to-upgrade-from 2023.2 ``` -## macOS - -Here is an example of how to create a new OS configuration (on Apple Silicon) and then run all -tests: +#### macOS ```bash -# Download some VM image -tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base - -# Create or edit configuration -# Use SSH to deploy the test runner since the image doesn't contain a runner -cargo run --bin test-manager set macos-ventura tart ventura-base macos \ - --architecture aarch64 \ - --provisioner ssh --ssh-user admin --ssh-password admin - -# Try it out to see if it works -#cargo run -p test-manager run-vm macos-ventura - # Run all tests -cargo run --bin test-manager run-tests macos-ventura \ +cargo run --bin test-manager run-tests --vm macos-ventura \ --display \ --account 0123456789 \ --app-package \ @@ -147,5 +166,4 @@ cargo run --bin test-manager run-tests macos-ventura \ ## Note on `scripts/ci-runtests.sh` -Account tokens are read (newline-delimited) from the path specified by the environment variable -`ACCOUNT_TOKENS`. Round robin is used to select an account for each VM. +`scripts/ci-runtests.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account tokens are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.