From c3f2a1d353890239ce0e3290831ff2f328c7ec3d Mon Sep 17 00:00:00 2001 From: Chris Awad Date: Fri, 15 Dec 2023 19:10:35 -0500 Subject: [PATCH 1/3] Fixed ek packet parsing when there are repeated protocol layers With ek, when protocols are repeated, the layer becomes a list of dictionaries, one for each layer. This update adds proper support for this. --- src/pyshark/tshark/output_parser/tshark_ek.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/pyshark/tshark/output_parser/tshark_ek.py b/src/pyshark/tshark/output_parser/tshark_ek.py index 5b6aeef5..fb98db6f 100644 --- a/src/pyshark/tshark/output_parser/tshark_ek.py +++ b/src/pyshark/tshark/output_parser/tshark_ek.py @@ -42,17 +42,27 @@ def packet_from_ek_packet(json_pkt): pkt_dict = json.loads(json_pkt.decode('utf-8')) # We use the frame dict here and not the object access because it's faster. - frame_dict = pkt_dict['layers'].pop('frame') - layers = [] - for layer in frame_dict['frame_frame_protocols'].split(':'): - layer_dict = pkt_dict['layers'].pop(layer, None) - if layer_dict is not None: - layers.append(EkLayer(layer, layer_dict)) + layers = pkt_dict['layers'] + frame_dict = layers.pop('frame') + + # Sort the frame protocol layers first + ek_layers = [] + for name in frame_dict['frame_frame_protocols'].split(':'): + layer = layers.get(name) + if isinstance(layer, list): + ek_layers.append(EkLayer(name, layer.pop(0))) + elif layers.pop(name, None) is not None: + ek_layers.append(EkLayer(name, layer)) + # Add all leftovers - for name, layer in pkt_dict['layers'].items(): - layers.append(EkLayer(name, layer)) + for name, layer in layers.items(): + if isinstance(layer, list): + for sub_layer in layer: + ek_layers.append(EkLayer(name, sub_layer) ) + else: + ek_layers.append(EkLayer(name, layer)) - return Packet(layers=layers, frame_info=EkLayer('frame', frame_dict), + return Packet(layers=ek_layers, frame_info=EkLayer('frame', frame_dict), number=int(frame_dict.get('frame_frame_number', 0)), length=int(frame_dict['frame_frame_len']), sniff_time=frame_dict['frame_frame_time_epoch'], From 8c18e4c7fc26338adc0abee8937c39d04d680e4f Mon Sep 17 00:00:00 2001 From: Chris Awad Date: Wed, 3 Jan 2024 17:59:39 -0500 Subject: [PATCH 2/3] Added check for empty list when traversing the frame.protocols in ek parsing Sometimes a frame layer doesn't actually exist in the packet details, so I added extra protection --- src/pyshark/tshark/output_parser/tshark_ek.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pyshark/tshark/output_parser/tshark_ek.py b/src/pyshark/tshark/output_parser/tshark_ek.py index fb98db6f..a223f71c 100644 --- a/src/pyshark/tshark/output_parser/tshark_ek.py +++ b/src/pyshark/tshark/output_parser/tshark_ek.py @@ -50,7 +50,8 @@ def packet_from_ek_packet(json_pkt): for name in frame_dict['frame_frame_protocols'].split(':'): layer = layers.get(name) if isinstance(layer, list): - ek_layers.append(EkLayer(name, layer.pop(0))) + if layer: + ek_layers.append(EkLayer(name, layer.pop(0))) elif layers.pop(name, None) is not None: ek_layers.append(EkLayer(name, layer)) From 733f3886d31ba75372738503a2664e9f859898f5 Mon Sep 17 00:00:00 2001 From: Chris Awad Date: Thu, 4 Jan 2024 21:58:11 +0000 Subject: [PATCH 3/3] Moved XXX_raw into XXX layers and fixed bug with packet.get_raw_packet() which didnt work --- src/pyshark/packet/packet.py | 4 +-- src/pyshark/tshark/output_parser/tshark_ek.py | 28 +++++++++++++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/pyshark/packet/packet.py b/src/pyshark/packet/packet.py index 627a196e..52a3f016 100644 --- a/src/pyshark/packet/packet.py +++ b/src/pyshark/packet/packet.py @@ -66,11 +66,11 @@ def __dir__(self): return dir(type(self)) + list(self.__dict__.keys()) + [l.layer_name for l in self.layers] def get_raw_packet(self) -> bytes: - assert "FRAME_RAW" in self, "Packet contains no raw data. In order to contains it, " \ + assert self.frame_info.has_field('raw'), "Packet contains no raw data. In order to contains it, " \ "make sure that use_json and include_raw are set to True " \ "in the Capture object" raw_packet = b'' - byte_values = [''.join(x) for x in zip(self.frame_raw.value[0::2], self.frame_raw.value[1::2])] + byte_values = [''.join(x) for x in zip(self.frame_info.raw[0::2], self.frame_info.raw[1::2])] for value in byte_values: raw_packet += binascii.unhexlify(value) return raw_packet diff --git a/src/pyshark/tshark/output_parser/tshark_ek.py b/src/pyshark/tshark/output_parser/tshark_ek.py index a223f71c..e137ad5e 100644 --- a/src/pyshark/tshark/output_parser/tshark_ek.py +++ b/src/pyshark/tshark/output_parser/tshark_ek.py @@ -34,8 +34,7 @@ def _extract_packet_from_data(self, data, got_first_packet=True): return data[start_index:linesep_location], data[linesep_location + 1:] - -def packet_from_ek_packet(json_pkt): +def packet_from_ek_packet_new(json_pkt): if USE_UJSON: pkt_dict = ujson.loads(json_pkt) else: @@ -44,17 +43,28 @@ def packet_from_ek_packet(json_pkt): # We use the frame dict here and not the object access because it's faster. layers = pkt_dict['layers'] frame_dict = layers.pop('frame') + if 'frame_raw' in layers: + frame_dict['frame_frame_raw'] = layers.pop('frame_raw') # Sort the frame protocol layers first ek_layers = [] for name in frame_dict['frame_frame_protocols'].split(':'): - layer = layers.get(name) - if isinstance(layer, list): - if layer: - ek_layers.append(EkLayer(name, layer.pop(0))) - elif layers.pop(name, None) is not None: - ek_layers.append(EkLayer(name, layer)) - + raw_name = f"{name}_raw" + if name in layers: + layer = layers.get(name) + layer_raw = layers.get(raw_name) + if not layer: + continue + elif isinstance(layer, list): + layer = layer.pop(0) + layer_raw = layer_raw.pop(0) if layer_raw else None + else: + layers.pop(name, None) + layers.pop(raw_name, None) + layer[f"{name}_{raw_name}"] = layer_raw + ek_layer = EkLayer(name, layer) + ek_layers.append(ek_layer) + # Add all leftovers for name, layer in layers.items(): if isinstance(layer, list):