[823] add detailed disk metrics

CHANGELOG.md

@@ -5,6 +5,7 @@ Bob versions changelog
 ## [Unreleased]
 #### Added
 - Blob performes fsync if buffered bytes are larger than max_dirty_bytes_before_sync config param (#748)
+- Added detailed information about total, used and free space on every disk (#823)
 
 #### Changed
 - Use cargo workspace to declare dependencies to avoid their duplication (#821)

bob/src/api/mod.rs

@@ -1,5 +1,5 @@
 use crate::{
-    build_info::BuildInfo, hw_metrics_collector::DiskSpaceMetrics, server::Server as BobServer,
+    build_info::BuildInfo, server::Server as BobServer,
 };
 use axum::{
     body::{self, BoxBody},
@@ -20,7 +20,7 @@ use bob_backend::pearl::{Group as PearlGroup, Holder, NoopHooks};
 use bob_common::{
     configs::node::TLSConfig,
     data::{BobData, BobKey, BobMeta, BOB_KEY_SIZE},
-    core_types::{VDisk as DataVDisk, NodeDisk},
+    core_types::{VDisk as DataVDisk, NodeDisk, DiskName},
     operation_options::{BobPutOptions, BobGetOptions, BobDeleteOptions},
     error::Error as BobError,
 };
@@ -145,12 +145,25 @@ pub(crate) struct NodeConfiguration {
 }
 
 #[derive(Debug, Serialize)]
-pub(crate) struct SpaceInfo {
+struct Space {
     total_disk_space_bytes: u64,
     free_disk_space_bytes: u64,
     used_disk_space_bytes: u64,
+}
+
+#[derive(Debug, Serialize)]
+pub(crate) struct SpaceInfo {
+    #[serde(flatten)]
+    total_space: Space,
+
+    /// Total occupied disk space by Bob
     occupied_disk_space_bytes: u64,
-    occupied_disk_space_by_disk: HashMap<String, u64>,
+
+    /// Key - Bob disk name
+    occupied_disk_space_by_disk: HashMap<DiskName, u64>,
+
+    /// Key - Mount point or Bob disk name if no mount point was found
+    disk_space_by_disk: HashMap<String, Space>,
 }
 
 async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer<RustlsAcceptor> {
@@ -313,31 +326,39 @@ async fn status<A: Authenticator>(bob: Extension<BobServer<A>>) -> Json<Node> {
 async fn get_space_info<A: Authenticator>(
     bob: Extension<BobServer<A>>,
 ) -> Result<Json<SpaceInfo>, StatusExt> {
-    let DiskSpaceMetrics {
-        total_space,
-        used_space,
-        free_space,
-    } = bob.grinder().hw_counter().update_space_metrics();
-
+    let disk_metrics = bob.grinder().hw_counter().update_space_metrics();
     let backend = bob.grinder().backend().inner();
     let (dcs, adc) = backend
         .disk_controllers()
         .ok_or_else(not_acceptable_backend)?;
-    let mut map = HashMap::new();
-    for dc in dcs.iter() {
-        map.insert(dc.disk().name().to_string(), dc.disk_used().await);
-    }
+
+    let mut occupied_disk_space_by_disk: HashMap<DiskName, u64> = futures::future::join_all(
+        dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) })
+    ).await.into_iter().collect();
     let adc_space = adc.disk_used().await;
-    map.entry(adc.disk().name().to_string())
-        .and_modify(|s| *s = *s + adc_space)
+
+    let disk_space_by_disk = disk_metrics.per_disk.into_iter().map(|(mount_point, disk)| (
+        mount_point.to_string_lossy().to_string(),
+        Space {
+            total_disk_space_bytes: disk.total_space,
+            free_disk_space_bytes: disk.free_space,
+            used_disk_space_bytes: disk.used_space 
+        }
+    )).collect();
+
+    occupied_disk_space_by_disk.entry(adc.disk().name().clone())
+        .and_modify(|s| *s += adc_space)
         .or_insert(adc_space);
 
     Ok(Json(SpaceInfo {
-        total_disk_space_bytes: total_space,
-        used_disk_space_bytes: used_space,
-        free_disk_space_bytes: free_space,
-        occupied_disk_space_bytes: map.values().sum(),
-        occupied_disk_space_by_disk: map,
+        total_space: Space {
+            total_disk_space_bytes: disk_metrics.total.total_space,
+            used_disk_space_bytes: disk_metrics.total.used_space,
+            free_disk_space_bytes: disk_metrics.total.free_space,
+        },
+        occupied_disk_space_bytes: occupied_disk_space_by_disk.values().sum(),
+        occupied_disk_space_by_disk,
+        disk_space_by_disk,
     }))
 }
 

bob/src/hw_metrics_collector.rs

@@ -6,6 +6,8 @@ use bob_common::metrics::{
     TOTAL_SPACE, FREE_SPACE, USED_SPACE, HW_DISKS_FOLDER
 };
 use libc::statvfs;
+use std::ops::{AddAssign, Add};
+use std::iter::Sum;
 use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 use tokio::process::Command;
@@ -16,7 +18,14 @@ const DESCRS_DIR: &str = "/proc/self/fd/";
 const CPU_STAT_FILE: &str = "/proc/stat";
 const DISK_STAT_FILE: &str = "/proc/diskstats";
 
+#[derive(Debug, Clone)]
 pub(crate) struct DiskSpaceMetrics {
+    pub(crate) total: SpaceMetrics,
+    pub(crate) per_disk: HashMap<PathBuf, SpaceMetrics>
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SpaceMetrics {
     pub(crate) total_space: u64,
     pub(crate) used_space: u64,
     pub(crate) free_space: u64,
@@ -63,6 +72,7 @@ impl HWMetricsCollector {
         tokio::spawn(Self::task(self.interval_time, self.disks.clone()));
     }
 
+    /// Returns the updated space metrics of this [`HWMetricsCollector`].
     pub(crate) fn update_space_metrics(&self) -> DiskSpaceMetrics {
         Self::update_space_metrics_from_disks(&self.disks)
     }
@@ -125,12 +135,22 @@ impl HWMetricsCollector {
         }
     }
 
-    fn update_space_metrics_from_disks(disks: &HashMap<PathBuf, DiskName>) -> DiskSpaceMetrics {
+    fn update_space_metrics_from_disks(
+        disks: &HashMap<PathBuf, DiskName>,
+    ) -> DiskSpaceMetrics {
         let disks_metrics = Self::space(disks);
-        gauge!(TOTAL_SPACE, bytes_to_mb(disks_metrics.total_space) as f64);
-        gauge!(USED_SPACE, bytes_to_mb(disks_metrics.used_space) as f64);
-        gauge!(FREE_SPACE, bytes_to_mb(disks_metrics.free_space) as f64);
-        disks_metrics
+        let SpaceMetrics { total_space, used_space, free_space } = disks_metrics.values().sum();
+        gauge!(TOTAL_SPACE, bytes_to_mb(total_space) as f64);
+        gauge!(USED_SPACE, bytes_to_mb(used_space) as f64);
+        gauge!(FREE_SPACE, bytes_to_mb(free_space) as f64);
+        DiskSpaceMetrics {
+                total: SpaceMetrics {
+                    total_space,
+                    used_space,
+                    free_space,
+                },
+                per_disk: disks_metrics,
+            }
     }
 
     fn to_cpath(path: &Path) -> Vec<u8> {
@@ -158,30 +178,35 @@ impl HWMetricsCollector {
     // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big,
     // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one
 
-    fn space(disks: &HashMap<PathBuf, DiskName>) -> DiskSpaceMetrics {
-        let mut total = 0;
-        let mut used = 0;
-        let mut free = 0;
-
-        for mount_point in disks.keys() {
+    /// Maps mount point to the corresponding [`SpaceMetrics`].
+    ///
+    /// Key -- mount points of used disks
+    /// Value -- Up-to-date [`SpaceMetrics`]
+    fn space(disks: &HashMap<PathBuf, DiskName>) -> HashMap<PathBuf, SpaceMetrics> {
+        let mut res = HashMap::new();
+        let mut fs_ids = HashSet::new();
+        for (mount_point, _) in disks {
             let cm_p = Self::to_cpath(mount_point.as_path());
             let stat = Self::statvfs_wrap(&cm_p);
             if let Some(stat) = stat {
-                let bsize = stat.f_bsize as u64;
-                let blocks = stat.f_blocks as u64;
-                let bavail = stat.f_bavail as u64;
-                let bfree = stat.f_bfree as u64;
-                total += bsize * blocks;
-                free += bsize * bavail;
-                used += (blocks - bfree) * bsize;
+                let bsize: u64 = stat.f_bsize;
+                let blocks: u64 = stat.f_blocks;
+                let bavail: u64 = stat.f_bavail;
+                let bfree: u64 = stat.f_bfree;
+                if fs_ids.insert(stat.f_fsid) { 
+                    res.insert(
+                        mount_point.clone(),
+                        SpaceMetrics {
+                            total_space: bsize * blocks,
+                            used_space: bsize * bavail,
+                            free_space: (blocks - bfree) * bsize,
+                        },
+                    );
+                }
             }
         }
 
-        DiskSpaceMetrics {
-            total_space: total,
-            used_space: used,
-            free_space: free,
-        }
+        res
     }
 }
 
@@ -603,3 +628,42 @@ async fn parse_command_output(command: &mut Command) -> Result<String, String> {
         }
     }
 }
+
+// Std Traits Impls
+
+impl<'a> AddAssign<&'a Self> for SpaceMetrics {
+    fn add_assign(&mut self, rhs: &Self) {
+        self.used_space += rhs.used_space;
+        self.free_space += rhs.free_space;
+        self.total_space += rhs.total_space;
+    }
+}
+impl<'a> Add<&'a Self> for SpaceMetrics {
+    type Output = Self;
+
+    fn add(mut self, rhs: &Self) -> Self::Output {
+        self += rhs;
+        self
+    }
+}
+
+impl<'a> Sum<&'a Self> for SpaceMetrics {
+    /// Summarize [`SpaceMetrics`] over an iterator.
+    /// NOTE: the `disk_name` field will be chosen from the first appeared disk in iterator if there
+    /// is any. Otherwise 'None' will be passed
+    fn sum<I: Iterator<Item = &'a Self>>(mut iter: I) -> Self {
+        let init = if let Some(metrics) = iter.next() {
+            metrics.clone()
+        } else {
+            return Self {
+                total_space: 0,
+                used_space: 0,
+                free_space: 0,
+            }
+        };
+        iter.fold(
+            init,
+            |a, b| a + b,
+        )
+    }
+}

config-examples/openapi.yaml

@@ -750,21 +750,51 @@ components:
           type: string
         root_dir_name:
           type: string
-    SpaceInfo:
+    Space:
       type: object
+      required:
+      - total_disk_space_bytes
+      - free_disk_space_bytes
+      - used_disk_space_bytes
       properties:
-        total_disk_space_bytes:
-          type: integer
         free_disk_space_bytes:
           type: integer
-        used_disk_space_bytes:
+          format: int64
+          minimum: 0
+        total_disk_space_bytes:
           type: integer
-        occupied_disk_space_bytes:
+          format: int64
+          minimum: 0
+        used_disk_space_bytes:
           type: integer
-        occupied_disk_space_by_disk:
-          type: object
-          additionalProperties:
+          format: int64
+          minimum: 0
+    SpaceInfo:
+      allOf:
+      - $ref: '#/components/schemas/Space'
+      - type: object
+        required:
+        - occupied_disk_space_bytes
+        - occupied_disk_space_by_disk
+        - disk_space_by_disk
+        properties:
+          disk_space_by_disk:
+            type: object
+            description: Key - Mount point or Bob disk name if no mount point was found
+            additionalProperties:
+              $ref: '#/components/schemas/Space'
+          occupied_disk_space_by_disk:
+            type: object
+            description: Key - Bob disk name
+            additionalProperties:
+              type: integer
+              format: int64
+              minimum: 0
+          occupied_disk_space_bytes:
             type: integer
+            format: int64
+            description: Total occupied disk space by Bob
+            minimum: 0
     MetricsEntryModel:
       type: object
       properties: