diff --git a/CHANGELOG.md b/CHANGELOG.md index 98b826591..391d9eca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Bob versions changelog ## [Unreleased] #### Added - Blob performes fsync if buffered bytes are larger than max_dirty_bytes_before_sync config param (#748) +- Added detailed information about total, used and free space on every disk (#823) #### Changed - Use cargo workspace to declare dependencies to avoid their duplication (#821) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 0c6aaab98..1879f0e53 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -1,5 +1,5 @@ use crate::{ - build_info::BuildInfo, hw_metrics_collector::DiskSpaceMetrics, server::Server as BobServer, + build_info::BuildInfo, server::Server as BobServer, }; use axum::{ body::{self, BoxBody}, @@ -20,7 +20,7 @@ use bob_backend::pearl::{Group as PearlGroup, Holder, NoopHooks}; use bob_common::{ configs::node::TLSConfig, data::{BobData, BobKey, BobMeta, BOB_KEY_SIZE}, - core_types::{VDisk as DataVDisk, NodeDisk}, + core_types::{VDisk as DataVDisk, NodeDisk, DiskName}, operation_options::{BobPutOptions, BobGetOptions, BobDeleteOptions}, error::Error as BobError, }; @@ -145,12 +145,25 @@ pub(crate) struct NodeConfiguration { } #[derive(Debug, Serialize)] -pub(crate) struct SpaceInfo { +struct Space { total_disk_space_bytes: u64, free_disk_space_bytes: u64, used_disk_space_bytes: u64, +} + +#[derive(Debug, Serialize)] +pub(crate) struct SpaceInfo { + #[serde(flatten)] + total_space: Space, + + /// Total occupied disk space by Bob occupied_disk_space_bytes: u64, - occupied_disk_space_by_disk: HashMap, + + /// Key - Bob disk name + occupied_disk_space_by_disk: HashMap, + + /// Key - Mount point or Bob disk name if no mount point was found + disk_space_by_disk: HashMap, } async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { @@ -313,31 +326,39 @@ async fn status(bob: Extension>) -> Json { async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = bob.grinder().hw_counter().update_space_metrics(); - + let disk_metrics = bob.grinder().hw_counter().update_space_metrics(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend .disk_controllers() .ok_or_else(not_acceptable_backend)?; - let mut map = HashMap::new(); - for dc in dcs.iter() { - map.insert(dc.disk().name().to_string(), dc.disk_used().await); - } + + let mut occupied_disk_space_by_disk: HashMap = futures::future::join_all( + dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) }) + ).await.into_iter().collect(); let adc_space = adc.disk_used().await; - map.entry(adc.disk().name().to_string()) - .and_modify(|s| *s = *s + adc_space) + + let disk_space_by_disk = disk_metrics.per_disk.into_iter().map(|(mount_point, disk)| ( + mount_point.to_string_lossy().to_string(), + Space { + total_disk_space_bytes: disk.total_space, + free_disk_space_bytes: disk.free_space, + used_disk_space_bytes: disk.used_space + } + )).collect(); + + occupied_disk_space_by_disk.entry(adc.disk().name().clone()) + .and_modify(|s| *s += adc_space) .or_insert(adc_space); Ok(Json(SpaceInfo { - total_disk_space_bytes: total_space, - used_disk_space_bytes: used_space, - free_disk_space_bytes: free_space, - occupied_disk_space_bytes: map.values().sum(), - occupied_disk_space_by_disk: map, + total_space: Space { + total_disk_space_bytes: disk_metrics.total.total_space, + used_disk_space_bytes: disk_metrics.total.used_space, + free_disk_space_bytes: disk_metrics.total.free_space, + }, + occupied_disk_space_bytes: occupied_disk_space_by_disk.values().sum(), + occupied_disk_space_by_disk, + disk_space_by_disk, })) } diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 3365c1fee..b0fd1808f 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -6,6 +6,8 @@ use bob_common::metrics::{ TOTAL_SPACE, FREE_SPACE, USED_SPACE, HW_DISKS_FOLDER }; use libc::statvfs; +use std::ops::{AddAssign, Add}; +use std::iter::Sum; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use tokio::process::Command; @@ -16,7 +18,14 @@ const DESCRS_DIR: &str = "/proc/self/fd/"; const CPU_STAT_FILE: &str = "/proc/stat"; const DISK_STAT_FILE: &str = "/proc/diskstats"; +#[derive(Debug, Clone)] pub(crate) struct DiskSpaceMetrics { + pub(crate) total: SpaceMetrics, + pub(crate) per_disk: HashMap +} + +#[derive(Debug, Clone)] +pub(crate) struct SpaceMetrics { pub(crate) total_space: u64, pub(crate) used_space: u64, pub(crate) free_space: u64, @@ -63,6 +72,7 @@ impl HWMetricsCollector { tokio::spawn(Self::task(self.interval_time, self.disks.clone())); } + /// Returns the updated space metrics of this [`HWMetricsCollector`]. pub(crate) fn update_space_metrics(&self) -> DiskSpaceMetrics { Self::update_space_metrics_from_disks(&self.disks) } @@ -125,12 +135,22 @@ impl HWMetricsCollector { } } - fn update_space_metrics_from_disks(disks: &HashMap) -> DiskSpaceMetrics { + fn update_space_metrics_from_disks( + disks: &HashMap, + ) -> DiskSpaceMetrics { let disks_metrics = Self::space(disks); - gauge!(TOTAL_SPACE, bytes_to_mb(disks_metrics.total_space) as f64); - gauge!(USED_SPACE, bytes_to_mb(disks_metrics.used_space) as f64); - gauge!(FREE_SPACE, bytes_to_mb(disks_metrics.free_space) as f64); - disks_metrics + let SpaceMetrics { total_space, used_space, free_space } = disks_metrics.values().sum(); + gauge!(TOTAL_SPACE, bytes_to_mb(total_space) as f64); + gauge!(USED_SPACE, bytes_to_mb(used_space) as f64); + gauge!(FREE_SPACE, bytes_to_mb(free_space) as f64); + DiskSpaceMetrics { + total: SpaceMetrics { + total_space, + used_space, + free_space, + }, + per_disk: disks_metrics, + } } fn to_cpath(path: &Path) -> Vec { @@ -158,30 +178,35 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - fn space(disks: &HashMap) -> DiskSpaceMetrics { - let mut total = 0; - let mut used = 0; - let mut free = 0; - - for mount_point in disks.keys() { + /// Maps mount point to the corresponding [`SpaceMetrics`]. + /// + /// Key -- mount points of used disks + /// Value -- Up-to-date [`SpaceMetrics`] + fn space(disks: &HashMap) -> HashMap { + let mut res = HashMap::new(); + let mut fs_ids = HashSet::new(); + for (mount_point, _) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { - let bsize = stat.f_bsize as u64; - let blocks = stat.f_blocks as u64; - let bavail = stat.f_bavail as u64; - let bfree = stat.f_bfree as u64; - total += bsize * blocks; - free += bsize * bavail; - used += (blocks - bfree) * bsize; + let bsize: u64 = stat.f_bsize; + let blocks: u64 = stat.f_blocks; + let bavail: u64 = stat.f_bavail; + let bfree: u64 = stat.f_bfree; + if fs_ids.insert(stat.f_fsid) { + res.insert( + mount_point.clone(), + SpaceMetrics { + total_space: bsize * blocks, + used_space: (blocks - bfree) * bsize, + free_space: bsize * bavail, + }, + ); + } } } - DiskSpaceMetrics { - total_space: total, - used_space: used, - free_space: free, - } + res } } @@ -603,3 +628,42 @@ async fn parse_command_output(command: &mut Command) -> Result { } } } + +// Std Traits Impls + +impl<'a> AddAssign<&'a Self> for SpaceMetrics { + fn add_assign(&mut self, rhs: &Self) { + self.used_space += rhs.used_space; + self.free_space += rhs.free_space; + self.total_space += rhs.total_space; + } +} +impl<'a> Add<&'a Self> for SpaceMetrics { + type Output = Self; + + fn add(mut self, rhs: &Self) -> Self::Output { + self += rhs; + self + } +} + +impl<'a> Sum<&'a Self> for SpaceMetrics { + /// Summarize [`SpaceMetrics`] over an iterator. + /// NOTE: the `disk_name` field will be chosen from the first appeared disk in iterator if there + /// is any. Otherwise 'None' will be passed + fn sum>(mut iter: I) -> Self { + let init = if let Some(metrics) = iter.next() { + metrics.clone() + } else { + return Self { + total_space: 0, + used_space: 0, + free_space: 0, + } + }; + iter.fold( + init, + |a, b| a + b, + ) + } +} diff --git a/config-examples/openapi.yaml b/config-examples/openapi.yaml index 899bf9974..958944e5c 100644 --- a/config-examples/openapi.yaml +++ b/config-examples/openapi.yaml @@ -750,21 +750,51 @@ components: type: string root_dir_name: type: string - SpaceInfo: + Space: type: object + required: + - total_disk_space_bytes + - free_disk_space_bytes + - used_disk_space_bytes properties: - total_disk_space_bytes: - type: integer free_disk_space_bytes: type: integer - used_disk_space_bytes: + format: int64 + minimum: 0 + total_disk_space_bytes: type: integer - occupied_disk_space_bytes: + format: int64 + minimum: 0 + used_disk_space_bytes: type: integer - occupied_disk_space_by_disk: - type: object - additionalProperties: + format: int64 + minimum: 0 + SpaceInfo: + allOf: + - $ref: '#/components/schemas/Space' + - type: object + required: + - occupied_disk_space_bytes + - occupied_disk_space_by_disk + - disk_space_by_disk + properties: + disk_space_by_disk: + type: object + description: Key - Mount point or Bob disk name if no mount point was found + additionalProperties: + $ref: '#/components/schemas/Space' + occupied_disk_space_by_disk: + type: object + description: Key - Bob disk name + additionalProperties: + type: integer + format: int64 + minimum: 0 + occupied_disk_space_bytes: type: integer + format: int64 + description: Total occupied disk space by Bob + minimum: 0 MetricsEntryModel: type: object properties: