From c096f93902312027beff1d684e75dbe86919351a Mon Sep 17 00:00:00 2001 From: Chengxiong Ruan Date: Tue, 28 May 2024 13:35:35 -0700 Subject: [PATCH] use stronger types to construct oomd config Summary: Use stronger typing for oomd config object and serialize with serde json at the end. It's till less than perfect because we still use String for most of the keys. But just want to introduce as little as possible changes to the json output. Differential Revision: D57761579 fbshipit-source-id: d2349d9c61745986a7478cf13d2ee0acd5651213 --- src/oomd/cfgen/src/cfgen.rs | 1176 +++++++++-------- src/oomd/cfgen/src/types.rs | 226 +++- .../test/cfgen_test_outputs/devvm/oomd2.json | 32 +- .../cfgen_test_outputs/ondemand/oomd2.json | 40 +- .../twshared_senpai_disabled/oomd2.json | 24 +- .../twshared_t20_zionex/oomd2.json | 28 +- .../twshared_vll_shard00/oomd2.json | 28 +- 7 files changed, 894 insertions(+), 660 deletions(-) diff --git a/src/oomd/cfgen/src/cfgen.rs b/src/oomd/cfgen/src/cfgen.rs index 3d09f53e..c95bb5e8 100644 --- a/src/oomd/cfgen/src/cfgen.rs +++ b/src/oomd/cfgen/src/cfgen.rs @@ -4,12 +4,13 @@ mod types; +use anyhow::anyhow; use libcfgen::prelude::*; use types::*; const CONFIG_VERSION: &str = "1.0.0"; -fn oomd_json(node: &Node) -> json::JsonValue { +fn oomd_json(node: &Node) -> Result { let attrs = get_attributes(node); match attrs.host_type { HostType::DevServer => devserver_json_config(node, &attrs), @@ -25,635 +26,646 @@ fn oomd_dropin(node: &Node) -> Dropin { .build(node) } -fn default_json_config(attrs: &ConfigParams) -> json::JsonValue { - let mut rulesets = json::Array::new(); - rulesets.push(rule_system_overview(attrs)); - rulesets.append(&mut rules_restart_cgroup_on_mem_threshold(attrs)); - rulesets.push(rule_protection_against_heavy_workload_thrashing(attrs)); +fn default_json_config(attrs: &ConfigParams) -> Result { + let mut rulesets = Vec::new(); + rulesets.push(ruleset_system_overview(attrs)); + rulesets.append(&mut rulesets_restart_cgroup_on_mem_threshold(attrs)); + rulesets.push(ruleset_protection_against_heavy_workload_thrashing(attrs)); if attrs.fbtax2.io_latency_supported { - rulesets.push(rule_protection_against_wdb_io_thrashing(attrs)); + rulesets.push(ruleset_protection_against_wdb_io_thrashing(attrs)); } if !attrs.fbtax2.disable_swap_protection { - rulesets.push(rule_fbtax2_protection_against_low_swap(attrs)); + rulesets.push(ruleset_fbtax2_protection_against_low_swap(attrs)); } if attrs.senpai.target.is_some() { - rulesets.push(rule_senpai_ruleset(attrs)); + rulesets.push(ruleset_senpai_ruleset(attrs)); } rulesets.append(&mut attrs.fbtax2.oomd_extra_rulesets.clone()); - rulesets.push(rule_senpai_drop_in_ruleset(attrs)); - rulesets.push(rule_tw_container_drop_in_ruleset(attrs)); + rulesets.push(ruleset_senpai_drop_in_ruleset(attrs)); + rulesets.push(ruleset_tw_container_drop_in_ruleset(attrs)); - json::object! { - "rulesets": rulesets, - "version": CONFIG_VERSION, - } + let config = OomdConfig { + rulesets, + version: CONFIG_VERSION.to_string(), + }; + + serialize_oomd_config(&config) } -fn devserver_json_config(node: &Node, attrs: &ConfigParams) -> json::JsonValue { - let mut rulesets = json::Array::new(); - rulesets.push(rule_system_overview(attrs)); - rulesets.push(rule_user_session_protection(node, attrs)); +fn devserver_json_config( + node: &Node, + attrs: &ConfigParams, +) -> Result { + let mut rulesets = Vec::new(); + rulesets.push(ruleset_system_overview(attrs)); + rulesets.push(ruleset_user_session_protection(node, attrs)); if !attrs.oomd2.disable_swap_protection { - rulesets.push(rule_oomd2_protection_against_low_swap(attrs)); - } - rulesets.push(rule_senpai_drop_in_ruleset(attrs)); - rulesets.append(&mut rules_restart_cgroup_on_mem_threshold(attrs)); - rulesets.push(rule_tw_container_drop_in_ruleset(attrs)); - json::object! { - "rulesets": rulesets, - "version": CONFIG_VERSION, - } -} - -fn od_json_config(attrs: &ConfigParams) -> json::JsonValue { - let mut rulesets = json::Array::new(); - rulesets.push(rule_system_overview(attrs)); - rulesets.push(rule_protection_against_high_memory_pressure(attrs)); - rulesets.append(&mut rules_restart_cgroup_on_mem_threshold(attrs)); - rulesets.push(rule_senpai_drop_in_ruleset(attrs)); - rulesets.push(rule_od_protection_against_low_swap(attrs)); - json::object! { - "rulesets": rulesets, - "version": CONFIG_VERSION, - } -} - -fn rule_system_overview(attrs: &ConfigParams) -> json::JsonValue { - let mut rule = json::object! { - "name": "system overview", - "silence-logs": "engine", - "detectors": [ - [ - "records system stats", - { - "name": "dump_cgroup_overview", - "args": { - "cgroup": attrs.oomd2.oomd_target.as_str(), - } - } - ] - ], - "actions": [ - { - "name": "continue", - "args": {}, - } - ] + rulesets.push(ruleset_oomd2_protection_against_low_swap(attrs)); + } + rulesets.push(ruleset_senpai_drop_in_ruleset(attrs)); + rulesets.append(&mut rulesets_restart_cgroup_on_mem_threshold(attrs)); + rulesets.push(ruleset_tw_container_drop_in_ruleset(attrs)); + + let config = OomdConfig { + rulesets, + version: CONFIG_VERSION.to_string(), + }; + + serialize_oomd_config(&config) +} + +fn od_json_config(attrs: &ConfigParams) -> Result { + let mut rulesets = Vec::new(); + rulesets.push(ruleset_system_overview(attrs)); + rulesets.push(ruleset_protection_against_high_memory_pressure(attrs)); + rulesets.append(&mut rulesets_restart_cgroup_on_mem_threshold(attrs)); + rulesets.push(ruleset_senpai_drop_in_ruleset(attrs)); + rulesets.push(ruleset_od_protection_against_low_swap(attrs)); + + let config = OomdConfig { + rulesets, + version: CONFIG_VERSION.to_string(), + }; + + serialize_oomd_config(&config) +} + +fn serialize_oomd_config(config: &OomdConfig) -> Result { + let json_output_result = serde_json::to_string(config); + let json_output = match json_output_result { + Ok(json_str) => json_str, + Err(err) => return Err(anyhow!(err)), + }; + let parsed_json_result = json::parse(json_output.as_str()); + match parsed_json_result { + Ok(json_value) => Ok(json_value), + Err(err) => Err(anyhow!(err)), + } +} + +fn ruleset_system_overview(attrs: &ConfigParams) -> RuleSet { + let mut rule = RuleSet { + name: "system overview".to_string(), + silence_logs: Some("engine".to_string()), + drop_in: None, + detectors: vec![detector!( + detector_name!("records system stats"), + detector_rule!( + name: "dump_cgroup_overview", + args: detector_rule_args!( + cgroup: attrs.oomd2.oomd_target.clone() + ) + ) + )], + actions: vec![action!( + name: "continue", + args: action_args!() + )], + prekill_hook_timeout: None, }; if attrs.host_type == HostType::OnDemand { - rule["drop-in"] = json::object! { - "detectors": true, - "actions": true, - }; + rule.drop_in = Some(DropIn { + disable_on_drop_in: None, + detectors: Some(true), + actions: Some(true), + }); } - rule } -fn rules_restart_cgroup_on_mem_threshold(attrs: &ConfigParams) -> json::Array { +fn rulesets_restart_cgroup_on_mem_threshold(attrs: &ConfigParams) -> Vec { attrs .oomd2 .oomd_restart_threshold .iter() - .map(|(cgroup, params)| { - json::object! { - "name": format!("restart {} on memory threshold", cgroup), - "detectors":[ - [ - "memory usage above", - { - "name": attrs.oomd2.plugins["memory_above"].as_str(), - "args": { - "cgroup": cgroup.as_str(), - "threshold_anon": params.threshold.as_str(), - "duration": params.duration.as_str(), - } - } - ] - ], - "actions":[ - { - "name": "systemd_restart", - "args": { - "service": params.service_name.as_str(), - "post_action_delay": params.post_action_delay.as_str(), - "dry": "false", - } - } - ] - } + .map(|(cgroup, params)| RuleSet { + name: format!("restart {} on memory threshold", cgroup), + silence_logs: None, + drop_in: None, + detectors: vec![detector!( + detector_name!("memory usage above"), + detector_rule!( + name: attrs.oomd2.plugins["memory_above"].as_str(), + args: detector_rule_args!( + cgroup: cgroup.clone(), + threshold_anon: params.threshold.clone(), + duration: params.duration.clone() + ) + ) + )], + actions: vec![action!( + name: "systemd_restart", + args: action_args!( + service: params.service_name.as_str(), + post_action_delay: params.post_action_delay.as_str(), + dry: "false" + ) + )], + prekill_hook_timeout: None, }) - .collect::() + .collect::>() } -fn rule_protection_against_heavy_workload_thrashing(attrs: &ConfigParams) -> json::JsonValue { - let mut action = json::object! { - "name": "kill_by_pg_scan", - "args": { - "cgroup": "workload.slice/workload-tw.slice/*", - "recursive": "true", - } - }; +fn ruleset_protection_against_heavy_workload_thrashing(attrs: &ConfigParams) -> RuleSet { + let mut action = action! ( + name: "kill_by_pg_scan", + args: action_args! ( + cgroup: "workload.slice/workload-tw.slice/*", + recursive: "true" + ) + ); if attrs.fbtax2.post_workload_kill_delay.is_some() { - action["args"]["post_action_delay"] = - json::JsonValue::String(attrs.fbtax2.post_workload_kill_delay.clone().unwrap()); + action.args.post_action_delay = attrs.fbtax2.post_workload_kill_delay.clone(); } - json::object! { - "name": "protection against heavy workload thrashing", - "drop-in": { - "disable-on-drop-in": true, - "detectors": true, - "actions": true, - }, - "detectors": rule_protection_against_heavy_workload_thrashing_detectors(attrs), - "actions": [ - action, - ] + RuleSet { + name: "protection against heavy workload thrashing".to_string(), + silence_logs: None, + drop_in: Some(DropIn { + disable_on_drop_in: Some(true), + detectors: Some(true), + actions: Some(true), + }), + detectors: ruleset_protection_against_heavy_workload_thrashing_detectors(attrs), + actions: vec![action], + prekill_hook_timeout: None, } } -fn rule_protection_against_heavy_workload_thrashing_detectors( +fn ruleset_protection_against_heavy_workload_thrashing_detectors( attrs: &ConfigParams, -) -> json::JsonValue { +) -> Vec { if attrs.fbtax2.io_cost_supported { - let mut detector = json::array! { - "sustained high workload memory pressure", - }; - - if !attrs.fbtax2.blacklisted_jobs.is_empty() { - _ = detector.push(fbtax2_blacklisted_jobs_detector_rule(attrs)); - } - - _ = detector.push(json::object! { - "name": "pressure_above", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "resource": "memory", - "threshold": attrs.fbtax2.workload_high_pressure_threshold.as_str(), - "duration": attrs.fbtax2.workload_high_pressure_duration.as_str(), - } - }); - - _ = detector.push(json::object! { - "name": "memory_reclaim", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "duration": "10", - } - }); - - return json::array! {detector}; - } - - let mut fast_gorwing_mem_pressure_detector = json::array! { - "detects fast growing memory pressure", - }; - - if !attrs.fbtax2.blacklisted_jobs.is_empty() { - _ = fast_gorwing_mem_pressure_detector.push(fbtax2_blacklisted_jobs_detector_rule(attrs)); - } - - _ = fast_gorwing_mem_pressure_detector.push(json::object! { - "name": "pressure_above", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "resource": "memory", - "threshold": "80", - "duration": "60", - } - }); - - _ = fast_gorwing_mem_pressure_detector.push(json::object! { - "name": "memory_reclaim", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "duration": "10", - } - }); - - let mut slow_growing_mem_pressure_detector = json::array! { - "detects slow growing memory pressure" - }; - - if !attrs.fbtax2.blacklisted_jobs.is_empty() { - _ = slow_growing_mem_pressure_detector.push(fbtax2_blacklisted_jobs_detector_rule(attrs)); - } - - _ = slow_growing_mem_pressure_detector.push(json::object! { - "name": "pressure_rising_beyond", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "resource": "memory", - "threshold": "60", - "duration": "90", - } - }); - - _ = slow_growing_mem_pressure_detector.push(json::object! { - "name": "memory_reclaim", - "args": { - "cgroup": attrs.fbtax2.workload_monitoring_slice.as_str(), - "duration": "10", - } - }); - - json::array! { - fast_gorwing_mem_pressure_detector, - slow_growing_mem_pressure_detector, - } -} - -fn fbtax2_blacklisted_jobs_detector_rule(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "exists", - "args": { - "cgroup": attrs.fbtax2.blacklisted_jobs.join(","), - "negate": true, - } - } -} - -fn rule_protection_against_wdb_io_thrashing(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "protection against wdb io thrashing", - "detectors": [ - [ - "low pressure in workload and high io in wdb", - { - "name": "pressure_rising_beyond", - "args": { - "cgroup": "workload.slice", - "resource": "io", - "threshold": if attrs.fbtax2.on_ssd {"10"} else {"15"}, - "duration": if attrs.fbtax2.on_ssd {"0"} else {"180"}, - } - }, - { - "name": "pressure_rising_beyond", - "args": { - "cgroup": "system.slice", - "resource": "io", - "threshold": if attrs.fbtax2.on_ssd {"60"} else {"85"}, - "duration": if attrs.fbtax2.on_ssd {"0"} else {"180"}, - } - } + vec![detector!( + detector_name!("sustained high workload memory pressure"), + if !attrs.fbtax2.blacklisted_jobs.is_empty() { + fbtax2_blacklisted_jobs_detector_rule(attrs) + } else { + skip_this_detector_rule!() + }, + detector_rule!( + name: "pressure_above", + args: detector_rule_args!( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + resource: "memory".to_string(), + threshold: attrs.fbtax2.workload_high_pressure_threshold.clone(), + duration: attrs.fbtax2.workload_high_pressure_duration.clone() + ) + ), + detector_rule!( + name: "memory_reclaim", + args: detector_rule_args! ( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + duration: "10".to_string() + ) + ) + )] + } else { + vec![ + detector!( + detector_name!("detects fast growing memory pressure"), + if !attrs.fbtax2.blacklisted_jobs.is_empty() { + fbtax2_blacklisted_jobs_detector_rule(attrs) + } else { + skip_this_detector_rule!() + }, + detector_rule!( + name: "pressure_above", + args: detector_rule_args!( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + resource: "memory".to_string(), + threshold: "80".to_string(), + duration: "60".to_string() + ) + ), + detector_rule!( + name: "memory_reclaim", + args: detector_rule_args! ( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + duration: "10".to_string() + ) + ) + ), + detector!( + detector_name!("detects slow growing memory pressure"), + if !attrs.fbtax2.blacklisted_jobs.is_empty() { + fbtax2_blacklisted_jobs_detector_rule(attrs) + } else { + skip_this_detector_rule!() + }, + detector_rule!( + name: "pressure_rising_beyond", + args: detector_rule_args!( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + resource: "memory".to_string(), + threshold: "60".to_string(), + duration: "90".to_string() + ) + ), + detector_rule!( + name: "memory_reclaim", + args: detector_rule_args! ( + cgroup: attrs.fbtax2.workload_monitoring_slice.clone(), + duration: "10".to_string() + ) + ) + ), ] - ], - "actions": [ - { - "name": "kill_by_pressure", - "args": { - "cgroup": "system.slice/*", - "recursive": "true", - "resource": "io", - } - } - ] } } -fn rule_fbtax2_protection_against_low_swap(attrs: &ConfigParams) -> json::JsonValue { - let mut detector = json::array! { - format!("free swap goes below {} percent", attrs.fbtax2.low_swap_threshold) - }; - - if !attrs.fbtax2.blacklisted_jobs.is_empty() { - _ = detector.push(fbtax2_blacklisted_jobs_detector_rule(attrs)); - } - - _ = detector.push(json::object! { - "name": "swap_free", - "args": { - "threshold_pct": attrs.fbtax2.low_swap_threshold.as_str(), - } - }); - - json::object! { - "name": "protection against low swap", - "detectors": [ - detector - ], - "actions": [ - { - "name": "kill_by_swap_usage", - "args": { - "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", - "biased_swap_kill": "true", - "recursive": "true", - } - } - ] - } -} - -fn rule_oomd2_protection_against_low_swap(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "protection against low swap", - "detectors": [ - [ - format!("free swap goes below {}%", attrs.oomd2.swap_protection_detect_threshold), - { - "name": "swap_free", - "args": { - "threshold_pct": attrs.oomd2.swap_protection_detect_threshold.as_str(), - } - } - ] - ], - "actions": [ - { - "name": "kill_by_swap_usage", - "args": { - "cgroup": attrs.oomd2.kill_target.as_str(), - "threshold": attrs.oomd2.swap_protection_kill_threshold.as_str(), - "recursive": true, - } - } - ] - } -} - -fn rule_senpai_ruleset(attrs: &ConfigParams) -> json::JsonValue { - let mut action_args = json::object! { - "io_pressure_pct": attrs.senpai.io_pressure_pct.as_str(), - "memory_high_timeout_ms": attrs.senpai.memory_high_timeout_ms.as_str(), - "scuba_logger_dataset": attrs.senpai.scuba_logger_dataset.as_str(), - }; +fn fbtax2_blacklisted_jobs_detector_rule(attrs: &ConfigParams) -> DetectorElement { + detector_rule!( + name: "exists", + args: detector_rule_args! ( + cgroup: attrs.fbtax2.blacklisted_jobs.join(","), + negate: true + ) + ) +} + +fn ruleset_protection_against_wdb_io_thrashing(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "protection against wdb io thrashing".to_string(), + silence_logs: None, + drop_in: None, + detectors: vec![detector!( + detector_name!("low pressure in workload and high io in wdb"), + detector_rule!( + name: "pressure_rising_beyond", + args: detector_rule_args!( + cgroup: "workload.slice".to_string(), + resource: "io".to_string(), + threshold: if attrs.fbtax2.on_ssd {"10".to_string()} else {"15".to_string()}, + duration: if attrs.fbtax2.on_ssd {"0".to_string()} else {"180".to_string()} + ) + ), + detector_rule!( + name: "pressure_rising_beyond", + args: detector_rule_args!( + cgroup: "workload.slice".to_string(), + resource: "io".to_string(), + threshold: if attrs.fbtax2.on_ssd {"60".to_string()} else {"85".to_string()}, + duration: if attrs.fbtax2.on_ssd {"0".to_string()} else {"180".to_string()} + ) + ) + )], + actions: vec![action!( + name: "kill_by_pressure", + args: action_args!( + cgroup: "system.slice/*", + recursive: "true", + resource: "io" + ) + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_fbtax2_protection_against_low_swap(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "protection against low swap".to_string(), + silence_logs: None, + drop_in: None, + detectors: vec![detector!( + detector_name!(format!( + "free swap goes below {} percent", + attrs.fbtax2.low_swap_threshold + )), + if !attrs.fbtax2.blacklisted_jobs.is_empty() { + fbtax2_blacklisted_jobs_detector_rule(attrs) + } else { + skip_this_detector_rule!() + }, + detector_rule!( + name: "swap_free", + args: detector_rule_args!( + threshold_pct: attrs.fbtax2.low_swap_threshold.clone() + ) + ) + )], + actions: vec![action!( + name: "kill_by_swap_usage", + args: action_args!( + cgroup: "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", + biased_swap_kill: "true", + recursive: "true" + ) + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_oomd2_protection_against_low_swap(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "protection against low swap".to_string(), + silence_logs: None, + drop_in: None, + detectors: vec![detector!( + detector_name!( + format!( + "free swap goes below {}%", + attrs.oomd2.swap_protection_detect_threshold + ) + .as_str() + ), + detector_rule!( + name: "swap_free", + args: detector_rule_args!( + threshold_pct: attrs.oomd2.swap_protection_detect_threshold.clone() + ) + ) + )], + actions: vec![action! ( + name: "kill_by_swap_usage", + args: action_args!( + cgroup: attrs.oomd2.kill_target.as_str(), + threshold: attrs.oomd2.swap_protection_kill_threshold.as_str(), + recursive: "true" + ) + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_senpai_ruleset(attrs: &ConfigParams) -> RuleSet { + let mut action_args = action_args!( + io_pressure_pct: attrs.senpai.io_pressure_pct.as_str(), + memory_high_timeout_ms: attrs.senpai.memory_high_timeout_ms.as_str(), + scuba_logger_dataset: attrs.senpai.scuba_logger_dataset.as_str() + ); if attrs.senpai.limit_min_bytes.is_some() { - action_args["limit_min_bytes"] = - json::JsonValue::String(attrs.senpai.limit_min_bytes.clone().unwrap()); + action_args.limit_min_bytes = attrs.senpai.limit_min_bytes.clone(); } if attrs.senpai.target.is_some() { - action_args["cgroup"] = json::JsonValue::String(attrs.senpai.target.clone().unwrap()); - } - - json::object! { - "name": "senpai ruleset", - "silence-logs": attrs.senpai.silence_logs.as_str(), - "detectors": [ - [ - "continue detector group", - { - "name": "continue", - "args": {} - } - ] - ], - "actions": [ - { - "name": "senpai_poking", - "args": action_args.clone(), - } - ] - } -} - -fn rule_senpai_drop_in_ruleset(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "senpai drop-in ruleset", - "silence-logs": if attrs.host_type == HostType::OnDemand {"engine,plugins"} else {"engine"}, - "drop-in": { - "actions": true, - "disable-on-drop-in": true, - }, - "detectors": [ - [ - if attrs.disable_senpai_dropin { - "stop detector group" - } else { - "continue detector group" - }, - if attrs.disable_senpai_dropin { - json::object! { - "name": "exists", - "args": { - "cgroup": "/", - "negate": true, - } + action_args.cgroup = attrs.senpai.target.clone(); + } + + RuleSet { + name: "senpai ruleset".to_string(), + silence_logs: Some(attrs.senpai.silence_logs.clone()), + drop_in: None, + detectors: vec![detector!( + detector_name!("continue detector group"), + detector_rule!( + name: "continue", + args: detector_rule_args!() + ) + )], + actions: vec![action!( + name: "senpai_poking", + args: action_args + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_senpai_drop_in_ruleset(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "senpai drop-in ruleset".to_string(), + silence_logs: Some(if attrs.host_type == HostType::OnDemand { + "engine,plugins".to_string() + } else { + "engine".to_string() + }), + drop_in: Some(DropIn { + disable_on_drop_in: Some(true), + actions: Some(true), + detectors: None, + }), + detectors: vec![detector!( + if attrs.disable_senpai_dropin { + detector_name!("stop detector group") + } else { + detector_name!("continue detector group") + }, + if attrs.disable_senpai_dropin { + detector_rule!( + name: "exists", + args: detector_rule_args!( + cgroup: "/".to_string(), + negate: true + ) + ) + } else { + detector_rule!( + name: "continue", + args: detector_rule_args!() + ) } - } else { - json::object! { - "name": "continue", - "args": {}, - } - } - ] - ], - "actions": [ - { - "name": "continue", - "args": {}, - } - ] - } -} - -fn rule_tw_container_drop_in_ruleset(attrs: &ConfigParams) -> json::JsonValue { - let mut rule = json::object! { - "name": "tw_container drop-in ruleset", - "drop-in": { - "detectors": true, - "actions": true, - "disable-on-drop-in": true - }, - "detectors": [ - [ - "continue", - { - "name": "stop", - "args": {} - } - ] - ], - "actions": [ - { - "name": "continue", - "args": {} - } - ], - }; - - if attrs.host_type != HostType::DevServer { - rule["prekill_hook_timeout"] = json::JsonValue::String(String::from("45")); + )], + actions: vec![action!( + name: "continue", + args: action_args!() + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_tw_container_drop_in_ruleset(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "tw_container drop-in ruleset".to_string(), + silence_logs: None, + drop_in: Some(DropIn { + detectors: Some(true), + actions: Some(true), + disable_on_drop_in: Some(true), + }), + detectors: vec![detector!( + detector_name!("continue"), + detector_rule!( + name: "stop", + args: detector_rule_args!() + ) + )], + actions: vec![action!( + name: "continue", + args: action_args!() + )], + prekill_hook_timeout: if attrs.host_type != HostType::DevServer { + Some("45".to_string()) + } else { + None + }, } - - rule } -fn rule_user_session_protection(node: &Node, attrs: &ConfigParams) -> json::JsonValue { - let mut user_pressure_detector = json::array! { - format!("user pressure above {} for 300s", attrs.devserver.user_mempress), - { - "name": "pressure_above", - "args": { - "cgroup": "user.slice,workload.slice,www.slice", - "resource": "memory", - "threshold": attrs.devserver.user_mempress.as_str(), - "duration": "300", - } - }, - }; - - let mut system_pressure_detector = json::array! { - format!("system pressure above {} for 300s", attrs.devserver.system_mempress), - { - "name": "pressure_above", - "args": { - "cgroup": "system.slice", - "resource": "memory", - "threshold": attrs.devserver.system_mempress.as_str(), - "duration": "300" - }, - } - }; - +fn ruleset_user_session_protection(node: &Node, attrs: &ConfigParams) -> RuleSet { + let user_pressure_detector = detector!( + detector_name!( + format!( + "user pressure above {} for 300s", + attrs.devserver.user_mempress + ) + .as_str() + ), + detector_rule!( + name: "pressure_above", + args: detector_rule_args!( + cgroup: "user.slice,workload.slice,www.slice".to_string(), + resource: "memory".to_string(), + threshold: attrs.devserver.user_mempress.clone(), + duration: "300".to_string() + ) + ), + maybe_nr_dying_descendants_rule(node), + detector_rule!( + name: "memory_reclaim", + args: detector_rule_args!( + cgroup: "user.slice,workload.slice,www.slice".to_string(), + duration: "30".to_string() + ) + ) + ); + + let system_pressure_detector = detector!( + detector_name!( + format!( + "system pressure above {} for 300s", + attrs.devserver.system_mempress + ) + .as_str() + ), + detector_rule!( + name: "pressure_above", + args: detector_rule_args!( + cgroup: "system.slice".to_string(), + resource: "memory".to_string(), + threshold: attrs.devserver.system_mempress.clone(), + duration: "300".to_string() + ) + ), + maybe_nr_dying_descendants_rule(node), + detector_rule!( + name: "memory_reclaim", + args: detector_rule_args!( + cgroup: "system.slice".to_string(), + duration: "30".to_string() + ) + ) + ); + + RuleSet { + name: "user session protection".to_string(), + silence_logs: None, + drop_in: None, + detectors: vec![user_pressure_detector, system_pressure_detector], + actions: vec![action!( + name: "kill_by_memory_size_or_growth", + args: action_args!( + cgroup: attrs.oomd2.kill_target.as_str(), + recursive: "true".to_string() + ) + )], + prekill_hook_timeout: None, + } +} + +fn maybe_nr_dying_descendants_rule(node: &Node) -> DetectorElement { if node.in_dynamic_smc_tier("devbig") { - _ = user_pressure_detector.push(json::object! { - "name": "nr_dying_descendants", - "args": { - "cgroup": "/", - "count": "30000", - "lte": "true" - } - }); - - _ = system_pressure_detector.push(json::object! { - "name": "nr_dying_descendants", - "args": { - "cgroup": "/", - "count": "30000", - "lte": "true" - } - }); - } - - _ = user_pressure_detector.push(json::object! { - "name": "memory_reclaim", - "args": { - "cgroup": "user.slice,workload.slice,www.slice", - "duration": "30" - } - }); - - _ = system_pressure_detector.push(json::object! { - "name": "memory_reclaim", - "args": { - "cgroup": "system.slice", - "duration": "30" - } - }); - - json::object! { - "name": "user session protection", - "detectors": [ - user_pressure_detector, - system_pressure_detector, - ], - "actions": [ - { - "name": "kill_by_memory_size_or_growth", - "args": { - "cgroup": attrs.oomd2.kill_target.as_str(), - "recursive": true, - } - } - ] - } -} - -fn rule_protection_against_high_memory_pressure(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "protection against high memory pressure", - "drop-in": { - "detectors": true, - "actions": true, - "disable-on-drop-in": attrs.oomd2.oomd_disable_on_drop_in, - }, - "detectors": [ - [ - "detects fast growing memory pressure", - { - "name": attrs.oomd2.plugins["pressure_above"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_target.as_str(), - "resource": "memory", - "threshold": attrs.oomd2.oomd_high_threshold.as_str(), - "duration": attrs.oomd2.oomd_high_threshold_duration.as_str(), - } - }, - { - "name": attrs.oomd2.plugins["memory_reclaim"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_target.as_str(), - "duration": attrs.oomd2.oomd_reclaim_duation.as_str(), - } - } + detector_rule!( + name: "nr_dying_descendants", + args: detector_rule_args!( + cgroup: "/".to_string(), + count: "30000".to_string(), + lte: "true".to_string() + ) + ) + } else { + skip_this_detector_rule!() + } +} + +fn ruleset_protection_against_high_memory_pressure(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "protection against high memory pressure".to_string(), + silence_logs: None, + drop_in: Some(DropIn { + detectors: Some(true), + actions: Some(true), + disable_on_drop_in: Some(attrs.oomd2.oomd_disable_on_drop_in), + }), + detectors: vec![ + detector!( + detector_name!("detects fast growing memory pressure"), + detector_rule!( + name: attrs.oomd2.plugins["pressure_above"].as_str(), + args: detector_rule_args!( + cgroup: attrs.oomd2.oomd_target.clone(), + resource: "memory".to_string(), + threshold: attrs.oomd2.oomd_high_threshold.clone(), + duration: attrs.oomd2.oomd_high_threshold_duration.clone() + ) + ), + detector_rule!( + name: attrs.oomd2.plugins["memory_reclaim"].as_str(), + args: detector_rule_args!( + cgroup: attrs.oomd2.oomd_target.clone(), + duration: attrs.oomd2.oomd_reclaim_duation.clone() + ) + ) + ), + detector!( + detector_name!("detects slow growing memory pressure"), + detector_rule!( + name: attrs.oomd2.plugins["pressure_rising_beyond"].as_str(), + args: detector_rule_args!( + cgroup: attrs.oomd2.oomd_target.clone(), + resource: "memory".to_string(), + threshold: attrs.oomd2.oomd_threshold.clone(), + duration: attrs.oomd2.oomd_threshold_duration.clone() + ) + ), + detector_rule!( + name: attrs.oomd2.plugins["memory_reclaim"].as_str(), + args: detector_rule_args!( + cgroup: attrs.oomd2.oomd_target.clone(), + duration: attrs.oomd2.oomd_reclaim_duation.clone() + ) + ) + ), ], - [ - "detects slow growing memory pressure", - { - "name": attrs.oomd2.plugins["pressure_rising_beyond"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_target.as_str(), - "resource": "memory", - "threshold": attrs.oomd2.oomd_threshold.as_str(), - "duration": attrs.oomd2.oomd_threshold_duration.as_str(), - } - }, - { - "name": attrs.oomd2.plugins["memory_reclaim"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_target.as_str(), - "duration": attrs.oomd2.oomd_reclaim_duation.as_str(), - } - } - ] - ], - "actions": [ - { - "name": attrs.oomd2.plugins["kill_by_memory_size_or_growth"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_action_target.as_str(), - "dry": if attrs.oomd2.oomd_dry { "true" } else {"false"}, - } - } - ] - } -} - -fn rule_od_protection_against_low_swap(attrs: &ConfigParams) -> json::JsonValue { - json::object! { - "name": "protection against low swap", - "drop-in": { - "detectors": true, - "actions": true, - "disable-on-drop-in": attrs.oomd2.oomd_disable_on_drop_in, - }, - "detectors": [ - [ - "free swap goes below 5 percent", - { - "name": attrs.oomd2.plugins["swap_free"].as_str(), - "args": { - "threshold_pct": "5", - } - } - ] - ], - "actions": [ - { - "name": attrs.oomd2.plugins["kill_by_swap_usage"].as_str(), - "args": { - "cgroup": attrs.oomd2.oomd_action_target.as_str(), - "dry": if attrs.oomd2.oomd_dry { "true" } else {"false"}, - } - } - ] + actions: vec![action!( + name: attrs.oomd2.plugins["kill_by_memory_size_or_growth"].as_str(), + args: action_args!( + cgroup: attrs.oomd2.oomd_action_target.as_str(), + dry: if attrs.oomd2.oomd_dry { "true" } else {"false"} + ) + )], + prekill_hook_timeout: None, + } +} + +fn ruleset_od_protection_against_low_swap(attrs: &ConfigParams) -> RuleSet { + RuleSet { + name: "protection against low swap".to_string(), + silence_logs: None, + drop_in: Some(DropIn { + detectors: Some(true), + actions: Some(true), + disable_on_drop_in: Some(attrs.oomd2.oomd_disable_on_drop_in), + }), + detectors: vec![detector!( + detector_name!("free swap goes below 5 percent"), + detector_rule!( + name: attrs.oomd2.plugins["swap_free"].as_str(), + args: detector_rule_args!( + threshold_pct: "5".to_string() + ) + ) + )], + actions: vec![action!( + name: attrs.oomd2.plugins["kill_by_swap_usage"].as_str(), + args: action_args!( + cgroup: attrs.oomd2.oomd_action_target.as_str(), + dry: if attrs.oomd2.oomd_dry { "true" } else {"false"} + ) + )], + prekill_hook_timeout: None, } } @@ -834,7 +846,7 @@ fn get_host_type(node: &Node) -> HostType { fn main() -> anyhow::Result<()> { let mut b = libcfgen::Builder::new(); - b = b.dynamic_json("oomd2.json", |node| Ok(oomd_json(node))); + b = b.dynamic_json("oomd2.json", oomd_json); b = b.dropin(|node| Ok(oomd_dropin(node))); b.run() } diff --git a/src/oomd/cfgen/src/types.rs b/src/oomd/cfgen/src/types.rs index 72ebb27e..841afcd8 100644 --- a/src/oomd/cfgen/src/types.rs +++ b/src/oomd/cfgen/src/types.rs @@ -1,6 +1,9 @@ use std::collections::BTreeMap; -use libcfgen::prelude::json::JsonValue; +use serde::Serialize; +use serde::Serializer; + +pub const DUMMY_DETECTOR_RULE_NAME: &str = "god_please_ignore_this_detector_rule_please"; #[derive(Debug, PartialEq, Eq)] pub enum HostType { @@ -29,7 +32,7 @@ pub struct FBTax2Attributes { pub workload_high_pressure_duration: String, pub workload_monitoring_slice: String, pub post_workload_kill_delay: Option, - pub oomd_extra_rulesets: Vec, + pub oomd_extra_rulesets: Vec, pub low_swap_threshold: String, } @@ -73,3 +76,222 @@ pub struct OomdRestartThreshold { pub post_action_delay: String, pub service_name: String, } + +#[derive(Serialize)] +pub struct OomdConfig { + pub rulesets: Vec, + pub version: String, +} + +#[derive(Clone, Serialize)] +pub struct RuleSet { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none", rename = "silence-logs")] + pub silence_logs: Option, + #[serde(skip_serializing_if = "Option::is_none", rename = "drop-in")] + pub drop_in: Option, // need logic to deserialize with name "drop-in" + pub detectors: Vec, + pub actions: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub prekill_hook_timeout: Option, +} + +pub type Detector = Vec; + +#[derive(Clone)] +pub enum DetectorElement { + String(String), + Rule(DetectorRule), +} + +impl Serialize for DetectorElement { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match *self { + DetectorElement::String(ref val) => serializer.serialize_str(val), + DetectorElement::Rule(ref val) => val.serialize(serializer), + } + } +} + +#[derive(Clone, Serialize)] +pub struct DetectorRule { + pub name: String, + pub args: DetectorRuleArgs, +} + +#[derive(Clone, Serialize)] +pub struct DetectorRuleArgs { + #[serde(skip_serializing_if = "Option::is_none")] + pub cgroup: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub lte: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold_anon: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub resource: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub negate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold_pct: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub count: Option, +} + +#[derive(Clone, Serialize)] +pub struct DropIn { + #[serde(skip_serializing_if = "Option::is_none", rename = "disable-on-drop-in")] + pub disable_on_drop_in: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub detectors: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub actions: Option, +} + +#[derive(Clone, Serialize)] +pub struct Action { + pub name: String, + pub args: ActionArgs, +} + +#[derive(Clone, Serialize)] +pub struct ActionArgs { + #[serde(skip_serializing_if = "Option::is_none")] + pub biased_swap_kill: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cgroup: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dry: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub post_action_delay: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub recursive: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub resource: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub service: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub io_pressure_pct: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_high_timeout_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scuba_logger_dataset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub limit_min_bytes: Option, +} + +#[macro_export] +macro_rules! detector { + ($($val:expr),*) => {{ + let mut vec = Vec::new(); + $( + match $val { + DetectorElement::String(_) => { + vec.push(DetectorElement::from($val)) + } + DetectorElement::Rule(e) => { + if e.name != types::DUMMY_DETECTOR_RULE_NAME { + vec.push(DetectorElement::from($val)); + } + } + } + )* + vec + }}; +} + +#[macro_export] +macro_rules! detector_name { + ($val:expr) => { + DetectorElement::String($val.to_string()) + }; +} + +#[macro_export] +macro_rules! detector_rule { + ($name:ident : $name_val:expr, $args:ident : $args_val:expr) => { + DetectorElement::Rule(DetectorRule { + $name: $name_val.to_string(), + $args: $args_val, + }) + }; +} + +#[macro_export] +macro_rules! skip_this_detector_rule { + () => { + DetectorElement::Rule(DetectorRule { + name: DUMMY_DETECTOR_RULE_NAME.to_string(), + args: detector_rule_args!(), + }) + }; +} + +#[macro_export] +macro_rules! detector_rule_args { + ($($name:ident : $val:expr),*) => {{ + #[allow(unused_mut)] + let mut ret = DetectorRuleArgs{ + cgroup: None, + duration: None, + lte: None, + threshold_anon: None, + resource: None, + threshold: None, + negate: None, + threshold_pct: None, + count: None, + }; + + $( + ret.$name = Some($val); + )* + + ret + }}; +} + +#[macro_export] +macro_rules! action { + ($name:ident : $name_val:expr, $args:ident : $args_val:expr) => { + Action { + $name: $name_val.to_string(), + $args: $args_val, + } + }; +} + +#[macro_export] +macro_rules! action_args { + ($($name:ident : $val:expr),*) => {{ + #[allow(unused_mut)] + let mut ret = ActionArgs{ + biased_swap_kill: None, + cgroup: None, + dry: None, + post_action_delay: None, + recursive: None, + resource: None, + service: None, + threshold: None, + io_pressure_pct: None, + memory_high_timeout_ms: None, + scuba_logger_dataset: None, + limit_min_bytes: None, + }; + + $( + ret.$name = Some($val.to_string()); + )* + + ret + }}; +} diff --git a/src/oomd/cfgen/test/cfgen_test_outputs/devvm/oomd2.json b/src/oomd/cfgen/test/cfgen_test_outputs/devvm/oomd2.json index a0943da1..13082f62 100644 --- a/src/oomd/cfgen/test/cfgen_test_outputs/devvm/oomd2.json +++ b/src/oomd/cfgen/test/cfgen_test_outputs/devvm/oomd2.json @@ -1,4 +1,4 @@ -@generated SignedSource<> +@generated SignedSource<<35a59b5bb6912fc9e9c91fe913a37146>> @codegen-command arc cfgen update-outputs fb-oomd { "rulesets": [ @@ -32,9 +32,9 @@ "name": "pressure_above", "args": { "cgroup": "user.slice,workload.slice,www.slice", + "duration": "300", "resource": "memory", - "threshold": "40", - "duration": "300" + "threshold": "40" } }, { @@ -51,9 +51,9 @@ "name": "pressure_above", "args": { "cgroup": "system.slice", + "duration": "300", "resource": "memory", - "threshold": "60", - "duration": "300" + "threshold": "60" } }, { @@ -70,7 +70,7 @@ "name": "kill_by_memory_size_or_growth", "args": { "cgroup": "user.slice/,system.slice/,workload.slice/,www.slice/", - "recursive": true + "recursive": "true" } } ] @@ -93,8 +93,8 @@ "name": "kill_by_swap_usage", "args": { "cgroup": "user.slice/,system.slice/,workload.slice/,www.slice/", - "threshold": "5", - "recursive": true + "recursive": "true", + "threshold": "5" } } ] @@ -103,8 +103,8 @@ "name": "senpai drop-in ruleset", "silence-logs": "engine", "drop-in": { - "actions": true, - "disable-on-drop-in": true + "disable-on-drop-in": true, + "actions": true }, "detectors": [ [ @@ -131,8 +131,8 @@ "name": "memory_above", "args": { "cgroup": "smc_proxy.service", - "threshold_anon": "15G", - "duration": "10" + "duration": "10", + "threshold_anon": "15G" } } ] @@ -141,9 +141,9 @@ { "name": "systemd_restart", "args": { - "service": "smc_proxy.service", + "dry": "false", "post_action_delay": "20", - "dry": "false" + "service": "smc_proxy.service" } } ] @@ -151,9 +151,9 @@ { "name": "tw_container drop-in ruleset", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [ diff --git a/src/oomd/cfgen/test/cfgen_test_outputs/ondemand/oomd2.json b/src/oomd/cfgen/test/cfgen_test_outputs/ondemand/oomd2.json index 4fda3f59..0f27dd3a 100644 --- a/src/oomd/cfgen/test/cfgen_test_outputs/ondemand/oomd2.json +++ b/src/oomd/cfgen/test/cfgen_test_outputs/ondemand/oomd2.json @@ -1,10 +1,14 @@ -@generated SignedSource<<60b23c1a40e237850d38e23d98d5a1bb>> +@generated SignedSource<<81eae96c318ecef1b5f4caf184cf4f37>> @codegen-command arc cfgen update-outputs fb-oomd { "rulesets": [ { "name": "system overview", "silence-logs": "engine", + "drop-in": { + "detectors": true, + "actions": true + }, "detectors": [ [ "records system stats", @@ -21,18 +25,14 @@ "name": "continue", "args": {} } - ], - "drop-in": { - "detectors": true, - "actions": true - } + ] }, { "name": "protection against high memory pressure", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [ @@ -41,9 +41,9 @@ "name": "pressure_above", "args": { "cgroup": "system.slice,workload.slice/workload-tw.slice/quicksand*.service", + "duration": "60", "resource": "memory", - "threshold": "80", - "duration": "60" + "threshold": "80" } }, { @@ -60,9 +60,9 @@ "name": "pressure_rising_beyond", "args": { "cgroup": "system.slice,workload.slice/workload-tw.slice/quicksand*.service", + "duration": "90", "resource": "memory", - "threshold": "60", - "duration": "90" + "threshold": "60" } }, { @@ -93,8 +93,8 @@ "name": "memory_above", "args": { "cgroup": "smc_proxy.service", - "threshold_anon": "15G", - "duration": "10" + "duration": "10", + "threshold_anon": "15G" } } ] @@ -103,9 +103,9 @@ { "name": "systemd_restart", "args": { - "service": "smc_proxy.service", + "dry": "false", "post_action_delay": "20", - "dry": "false" + "service": "smc_proxy.service" } } ] @@ -114,8 +114,8 @@ "name": "senpai drop-in ruleset", "silence-logs": "engine,plugins", "drop-in": { - "actions": true, - "disable-on-drop-in": true + "disable-on-drop-in": true, + "actions": true }, "detectors": [ [ @@ -139,9 +139,9 @@ { "name": "protection against low swap", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [ diff --git a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_senpai_disabled/oomd2.json b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_senpai_disabled/oomd2.json index 1e9fcc2f..f0a8d3cf 100644 --- a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_senpai_disabled/oomd2.json +++ b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_senpai_disabled/oomd2.json @@ -1,4 +1,4 @@ -@generated SignedSource<<2b86656315099f05b2d1754bef6c5220>> +@generated SignedSource<> @codegen-command arc cfgen update-outputs fb-oomd { "rulesets": [ @@ -32,8 +32,8 @@ "name": "memory_above", "args": { "cgroup": "smc_proxy.service", - "threshold_anon": "15G", - "duration": "10" + "duration": "10", + "threshold_anon": "15G" } } ] @@ -42,9 +42,9 @@ { "name": "systemd_restart", "args": { - "service": "smc_proxy.service", + "dry": "false", "post_action_delay": "20", - "dry": "false" + "service": "smc_proxy.service" } } ] @@ -70,9 +70,9 @@ "name": "pressure_above", "args": { "cgroup": "workload.slice/workload-tw.slice", + "duration": "180", "resource": "memory", - "threshold": "80", - "duration": "180" + "threshold": "80" } }, { @@ -118,8 +118,8 @@ { "name": "kill_by_swap_usage", "args": { - "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "biased_swap_kill": "true", + "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "recursive": "true" } } @@ -129,8 +129,8 @@ "name": "senpai drop-in ruleset", "silence-logs": "engine", "drop-in": { - "actions": true, - "disable-on-drop-in": true + "disable-on-drop-in": true, + "actions": true }, "detectors": [ [ @@ -151,9 +151,9 @@ { "name": "tw_container drop-in ruleset", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [ diff --git a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_t20_zionex/oomd2.json b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_t20_zionex/oomd2.json index d5379a34..c65bf849 100644 --- a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_t20_zionex/oomd2.json +++ b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_t20_zionex/oomd2.json @@ -1,4 +1,4 @@ -@generated SignedSource<> +@generated SignedSource<<321bf87df3cdbf0d9f32602db039bec7>> @codegen-command arc cfgen update-outputs fb-oomd { "rulesets": [ @@ -32,8 +32,8 @@ "name": "memory_above", "args": { "cgroup": "smc_proxy.service", - "threshold_anon": "15G", - "duration": "10" + "duration": "10", + "threshold_anon": "15G" } } ] @@ -42,9 +42,9 @@ { "name": "systemd_restart", "args": { - "service": "smc_proxy.service", + "dry": "false", "post_action_delay": "20", - "dry": "false" + "service": "smc_proxy.service" } } ] @@ -70,9 +70,9 @@ "name": "pressure_above", "args": { "cgroup": "workload.slice/workload-tw.slice", + "duration": "180", "resource": "memory", - "threshold": "80", - "duration": "180" + "threshold": "80" } }, { @@ -118,8 +118,8 @@ { "name": "kill_by_swap_usage", "args": { - "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "biased_swap_kill": "true", + "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "recursive": "true" } } @@ -141,11 +141,11 @@ { "name": "senpai_poking", "args": { + "cgroup": "system.slice,workload.slice/workload-wdb.slice,hostcritical.slice,workload.slice/workload-wdb.slice/*,hostcritical.slice/*", "io_pressure_pct": "1.0", "memory_high_timeout_ms": "20", "scuba_logger_dataset": "perfpipe_senpai_events", - "limit_min_bytes": "104857600", - "cgroup": "system.slice,workload.slice/workload-wdb.slice,hostcritical.slice,workload.slice/workload-wdb.slice/*,hostcritical.slice/*" + "limit_min_bytes": "104857600" } } ] @@ -154,8 +154,8 @@ "name": "senpai drop-in ruleset", "silence-logs": "engine", "drop-in": { - "actions": true, - "disable-on-drop-in": true + "disable-on-drop-in": true, + "actions": true }, "detectors": [ [ @@ -176,9 +176,9 @@ { "name": "tw_container drop-in ruleset", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [ diff --git a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_vll_shard00/oomd2.json b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_vll_shard00/oomd2.json index d5379a34..c65bf849 100644 --- a/src/oomd/cfgen/test/cfgen_test_outputs/twshared_vll_shard00/oomd2.json +++ b/src/oomd/cfgen/test/cfgen_test_outputs/twshared_vll_shard00/oomd2.json @@ -1,4 +1,4 @@ -@generated SignedSource<> +@generated SignedSource<<321bf87df3cdbf0d9f32602db039bec7>> @codegen-command arc cfgen update-outputs fb-oomd { "rulesets": [ @@ -32,8 +32,8 @@ "name": "memory_above", "args": { "cgroup": "smc_proxy.service", - "threshold_anon": "15G", - "duration": "10" + "duration": "10", + "threshold_anon": "15G" } } ] @@ -42,9 +42,9 @@ { "name": "systemd_restart", "args": { - "service": "smc_proxy.service", + "dry": "false", "post_action_delay": "20", - "dry": "false" + "service": "smc_proxy.service" } } ] @@ -70,9 +70,9 @@ "name": "pressure_above", "args": { "cgroup": "workload.slice/workload-tw.slice", + "duration": "180", "resource": "memory", - "threshold": "80", - "duration": "180" + "threshold": "80" } }, { @@ -118,8 +118,8 @@ { "name": "kill_by_swap_usage", "args": { - "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "biased_swap_kill": "true", + "cgroup": "system.slice/*,workload.slice/workload-wdb.slice/*,workload.slice/workload-tw.slice/*", "recursive": "true" } } @@ -141,11 +141,11 @@ { "name": "senpai_poking", "args": { + "cgroup": "system.slice,workload.slice/workload-wdb.slice,hostcritical.slice,workload.slice/workload-wdb.slice/*,hostcritical.slice/*", "io_pressure_pct": "1.0", "memory_high_timeout_ms": "20", "scuba_logger_dataset": "perfpipe_senpai_events", - "limit_min_bytes": "104857600", - "cgroup": "system.slice,workload.slice/workload-wdb.slice,hostcritical.slice,workload.slice/workload-wdb.slice/*,hostcritical.slice/*" + "limit_min_bytes": "104857600" } } ] @@ -154,8 +154,8 @@ "name": "senpai drop-in ruleset", "silence-logs": "engine", "drop-in": { - "actions": true, - "disable-on-drop-in": true + "disable-on-drop-in": true, + "actions": true }, "detectors": [ [ @@ -176,9 +176,9 @@ { "name": "tw_container drop-in ruleset", "drop-in": { + "disable-on-drop-in": true, "detectors": true, - "actions": true, - "disable-on-drop-in": true + "actions": true }, "detectors": [ [