Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[40] Http response fingerprinting support #45

Merged
merged 5 commits into from
Dec 26, 2024
Merged

[40] Http response fingerprinting support #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d7bba3b
adding http response in the interface
biandratti Dec 25, 2024
a8818b0
TcpFlow addapted to a sequence of data
biandratti Dec 26, 2024
f7f89b1
split server and client extract packages
biandratti Dec 26, 2024
55c6a04
remove package from cache after response from server
biandratti Dec 26, 2024
60665ab
unify p0f_output
biandratti Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions examples/p0f.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,5 +84,8 @@ fn main() {
if let Some(http_request) = output.http_request {
info!("{}", http_request);
}
if let Some(http_response) = output.http_response {
info!("{}", http_response);
}
}
}
181 changes: 137 additions & 44 deletions src/http_process.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,64 @@ use pnet::packet::ipv6::Ipv6Packet;
use pnet::packet::tcp::TcpPacket;
use pnet::packet::Packet;
use std::net::IpAddr;
use std::time::{Duration, Instant};
use std::time::Duration;
use ttl_cache::TtlCache;

/// FlowKey: (Client IP, Server IP, Client Port, Server Port)
pub type FlowKey = (IpAddr, IpAddr, u16, u16);

//TODO: WIP
#[allow(dead_code)]
#[derive(Clone)]
struct TcpData {
sequence: u32,
data: Vec<u8>,
}

pub struct TcpFlow {
client_ip: IpAddr,
server_ip: IpAddr,
client_port: u16,
server_port: u16,
client_seq: u32,
server_seq: u32,
client_data: Vec<u8>, // Aggregated HTTP request payload
server_data: Vec<u8>, // Aggregated HTTP response payload
last_seen: Instant, // Timestamp for flow expiration
client_data: Vec<TcpData>,
server_data: Vec<TcpData>,
}
impl TcpFlow {
fn init(
src_ip: IpAddr,
src_port: u16,
dst_ip: IpAddr,
dst_port: u16,
tcp_data: TcpData,
) -> TcpFlow {
TcpFlow {
client_ip: src_ip,
server_ip: dst_ip,
client_port: src_port,
server_port: dst_port,
client_data: vec![tcp_data.clone()],
server_data: Vec::new(),
}
}
/// Traversing all the data in sequence in the correct order to build the full data
///
/// # Parameters
/// - `is_client`: If the data comes from the client.
fn get_full_data(&self, is_client: bool) -> Vec<u8> {
let data = if is_client {
&self.client_data
} else {
&self.server_data
};

let mut sorted_data = data.clone();

sorted_data.sort_by_key(|tcp_data| tcp_data.sequence);

let mut full_data = Vec::new();
for tcp_data in sorted_data {
full_data.extend_from_slice(&tcp_data.data);
}
full_data
}
}

pub fn process_http_ipv4(
Expand All @@ -44,7 +84,10 @@ pub fn process_http_ipv4(
IpAddr::V4(packet.get_destination()),
)
} else {
Ok(ObservableHttpPackage { http_request: None })
Ok(ObservableHttpPackage {
http_request: None,
http_response: None,
})
}
}

Expand All @@ -63,7 +106,10 @@ pub fn process_http_ipv6(
IpAddr::V6(packet.get_destination()),
)
} else {
Ok(ObservableHttpPackage { http_request: None })
Ok(ObservableHttpPackage {
http_request: None,
http_response: None,
})
}
}

Expand All @@ -73,43 +119,44 @@ fn process_tcp_packet(
src_ip: IpAddr,
dst_ip: IpAddr,
) -> Result<ObservableHttpPackage, Error> {
let src_port = tcp.get_source();
let dst_port = tcp.get_destination();
let flow_key = (src_ip, dst_ip, src_port, dst_port);
let src_port: u16 = tcp.get_source();
let dst_port: u16 = tcp.get_destination();
let flow_key: FlowKey = (src_ip, dst_ip, src_port, dst_port);
let mut http_request: Option<ObservableHttpRequest> = None;
let mut http_response: Option<ObservableHttpResponse> = None;

if tcp.get_flags() & pnet::packet::tcp::TcpFlags::SYN != 0 {
let flow = TcpFlow {
client_ip: src_ip,
server_ip: dst_ip,
client_port: src_port,
server_port: dst_port,
client_seq: tcp.get_sequence(),
server_seq: 0,
client_data: Vec::new(),
server_data: Vec::new(),
last_seen: Instant::now(),
let tcp_data: TcpData = TcpData {
sequence: tcp.get_sequence(),
data: Vec::from(tcp.payload()),
};
let flow: TcpFlow = TcpFlow::init(src_ip, src_port, dst_ip, dst_port, tcp_data);
cache.insert(flow_key, flow, Duration::new(60, 0));
return Ok(ObservableHttpPackage { http_request: None });
}

if let Some(flow) = cache.get_mut(&flow_key) {
flow.last_seen = Instant::now();

if !tcp.payload().is_empty() {
// Append the new data to the flow data
if src_ip == flow.client_ip && src_port == flow.client_port {
flow.client_data.extend_from_slice(tcp.payload());
if let Ok(http_request_parsed) = parse_http_request(&flow.client_data) {
http_request = http_request_parsed;
}
} else {
// Try to parse the HTTP response when enough data is accumulated
/*flow.server_data.extend_from_slice(tcp.payload());
if let Ok(response) = parse_http_response(&flow.server_data) {
info!("HTTP Response:\n{:?}", response);
}*/
if !tcp.payload().is_empty() && src_ip == flow.client_ip && src_port == flow.client_port {
let tcp_data: TcpData = TcpData {
sequence: tcp.get_sequence(),
data: Vec::from(tcp.payload()),
};
flow.client_data.push(tcp_data);
if let Ok(http_request_parsed) = parse_http_request(&flow.get_full_data(true)) {
http_request = http_request_parsed;
}
}
}

let flow_key: FlowKey = (dst_ip, src_ip, dst_port, src_port);
if let Some(flow) = cache.get_mut(&flow_key) {
if !tcp.payload().is_empty() && src_ip == flow.server_ip && src_port == flow.server_port {
let tcp_data: TcpData = TcpData {
sequence: tcp.get_sequence(),
data: Vec::from(tcp.payload()),
};
flow.server_data.push(tcp_data);
if let Ok(http_response_parsed) = parse_http_response(&flow.get_full_data(false)) {
http_response = http_response_parsed;
}
}

Expand All @@ -121,7 +168,10 @@ fn process_tcp_packet(
}
}

Ok(ObservableHttpPackage { http_request })
Ok(ObservableHttpPackage {
http_request,
http_response,
})
}

fn parse_http_request(data: &[u8]) -> Result<Option<ObservableHttpRequest>, Error> {
Expand All @@ -137,13 +187,9 @@ fn parse_http_request(data: &[u8]) -> Result<Option<ObservableHttpRequest>, Erro
.collect();

let headers_in_order: Vec<Header> = build_headers_in_order(&headers);

let headers_absent: Vec<Header> = build_headers_absent_in_order(&headers);

let user_agent: Option<String> = extract_user_agent(&headers);

let lang: Option<String> = extract_accept_language(&headers);

let http_version: Version = extract_http_version(req);

Ok(Some(ObservableHttpRequest {
Expand Down Expand Up @@ -174,11 +220,54 @@ fn parse_http_request(data: &[u8]) -> Result<Option<ObservableHttpRequest>, Erro
}
}

fn parse_http_response(data: &[u8]) -> Result<Option<ObservableHttpResponse>, Error> {
let mut headers = [EMPTY_HEADER; 64];
let mut req = Request::new(&mut headers);

match req.parse(data) {
Ok(httparse::Status::Complete(_)) => {
let headers: Vec<Header> = req
.headers
.iter()
.map(|h| Header::new(h.name).with_value(String::from_utf8_lossy(h.value)))
.collect();

let headers_in_order: Vec<Header> = build_headers_in_order(&headers);
let headers_absent: Vec<Header> = build_headers_absent_in_order(&headers);
let http_version: Version = extract_http_version(req);

Ok(Some(ObservableHttpResponse {
signature: http::Signature {
version: http_version,
horder: headers_in_order,
habsent: headers_absent,
expsw: extract_traffic_classification(None),
},
}))
}
Ok(httparse::Status::Partial) => {
debug!("Incomplete HTTP response data. Data: {:?}", data);
Ok(None)
}
Err(e) => {
debug!(
"Failed to parse HTTP response with Data: {:?}. Error: {}",
data, e
);
Err(failure::err_msg(format!(
"Failed to parse HTTP response: {}",
e
)))
}
}
}

fn build_headers_in_order(headers: &[Header]) -> Vec<Header> {
// TODO: WIP
headers.to_vec()
}

// TODO: WIP
fn build_headers_absent_in_order(headers: &[Header]) -> Vec<Header> {
// List of expected headers
let expected_headers = [
Expand Down Expand Up @@ -229,6 +318,7 @@ fn extract_http_version(request: Request) -> Version {

pub struct ObservableHttpPackage {
pub http_request: Option<ObservableHttpRequest>,
pub http_response: Option<ObservableHttpResponse>,
}

#[derive(Debug)]
Expand All @@ -237,6 +327,9 @@ pub struct ObservableHttpRequest {
pub user_agent: Option<String>,
pub signature: http::Signature,
}
pub struct ObservableHttpResponse {
pub signature: http::Signature,
}

#[cfg(test)]
mod tests {
Expand Down
48 changes: 31 additions & 17 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ mod uptime;
use crate::db::Database;
use crate::http_process::{FlowKey, TcpFlow};
use crate::p0f_output::{
HttpRequestOutput, MTUOutput, P0fOutput, SynAckTCPOutput, SynTCPOutput, UptimeOutput,
HttpRequestOutput, HttpResponseOutput, MTUOutput, P0fOutput, SynAckTCPOutput, SynTCPOutput,
UptimeOutput,
};
use crate::process::ObservablePackage;
use crate::signature_matcher::SignatureMatcher;
Expand Down Expand Up @@ -119,7 +120,7 @@ impl<'a> P0f<'a> {
fn analyze_tcp(&mut self, packet: &[u8]) -> P0fOutput {
match ObservablePackage::extract(packet, &mut self.tcp_cache, &mut self.http_cache) {
Ok(observable_package) => {
let (syn, syn_ack, mtu, uptime, http_request) = {
let (syn, syn_ack, mtu, uptime, http_request, http_response) = {
let mtu: Option<MTUOutput> =
observable_package.mtu.and_then(|observable_mtu| {
self.matcher
Expand Down Expand Up @@ -169,22 +170,33 @@ impl<'a> P0f<'a> {
freq: update.freq,
});

let http_request =
observable_package
.http_request
.map(|http_request| HttpRequestOutput {
source: observable_package.source.clone(),
destination: observable_package.destination.clone(),
lang: http_request.lang,
user_agent: http_request.user_agent,
label: self
.matcher
.matching_by_http_request(&http_request.signature)
.map(|(label, _)| label.clone()),
sig: http_request.signature,
});
let http_request: Option<HttpRequestOutput> = observable_package
.http_request
.map(|http_request| HttpRequestOutput {
source: observable_package.source.clone(),
destination: observable_package.destination.clone(),
lang: http_request.lang,
user_agent: http_request.user_agent,
label: self
.matcher
.matching_by_http_request(&http_request.signature)
.map(|(label, _)| label.clone()),
sig: http_request.signature,
});

let http_response: Option<HttpResponseOutput> = observable_package
.http_response
.map(|http_response| HttpResponseOutput {
source: observable_package.source.clone(),
destination: observable_package.destination.clone(),
label: self
.matcher
.matching_by_http_response(&http_response.signature)
.map(|(label, _)| label.clone()),
sig: http_response.signature,
});

(syn, syn_ack, mtu, uptime, http_request)
(syn, syn_ack, mtu, uptime, http_request, http_response)
};

P0fOutput {
Expand All @@ -193,6 +205,7 @@ impl<'a> P0f<'a> {
mtu,
uptime,
http_request,
http_response,
}
}
Err(error) => {
Expand All @@ -203,6 +216,7 @@ impl<'a> P0f<'a> {
mtu: None,
uptime: None,
http_request: None,
http_response: None,
}
}
}
Expand Down
Loading
Loading