Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ECS support #91

Merged
merged 12 commits into from
Feb 4, 2024
Prev Previous commit
Next Next commit
Lint
andriilahuta committed Jan 29, 2024
commit 82af01762b1f74ed64ab6c5ca7d3a8767348d02e
3 changes: 2 additions & 1 deletion logstash_async/constants.py
Original file line number Diff line number Diff line change
@@ -44,7 +44,8 @@ class Constants:
FORMATTER_LOGSTASH_MESSAGE_FIELD_LIST = [
'@timestamp', '@version', 'host', 'level', 'logsource', 'message',
'pid', 'program', 'type', 'tags', '@metadata']
FORMATTER_LOGSTASH_ECS_MESSAGE_FIELD_LIST = ['@timestamp', '@version', '@metadata', 'message', 'labels', 'tags']
FORMATTER_LOGSTASH_ECS_MESSAGE_FIELD_LIST = [
'@timestamp', '@version', '@metadata', 'message', 'labels', 'tags']
# convert dotted ECS fields into nested objects
FORMATTER_LOGSTASH_ECS_NORMALIZE_MESSAGE = True
# enable rate limiting for error messages (e.g. network errors) emitted by the logger
14 changes: 10 additions & 4 deletions logstash_async/formatter.py
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@
from logstash_async.utils import normalize_ecs_dict
import logstash_async


try:
import json
except ImportError:
@@ -265,7 +266,8 @@ class LogstashEcsFormatter(LogstashFormatter):
}

normalize_ecs_message = constants.FORMATTER_LOGSTASH_ECS_NORMALIZE_MESSAGE
top_level_field_set = {*constants.FORMATTER_LOGSTASH_ECS_MESSAGE_FIELD_LIST, *__schema_dict.values()}
top_level_field_set = {*constants.FORMATTER_LOGSTASH_ECS_MESSAGE_FIELD_LIST,
*__schema_dict.values()}
MessageSchema = type('MessageSchema', (LogstashFormatter.MessageSchema,), __schema_dict)

def _get_primary_fields(self, record):
@@ -277,6 +279,7 @@ def _get_primary_fields(self, record):
def _format_to_dict(self, record):
message = super()._format_to_dict(record)
if self.normalize_ecs_message:
# pylint: disable-next=redefined-variable-type
message = normalize_ecs_dict(message)
return message

@@ -435,19 +438,22 @@ def __init__(self, *args, **kwargs):

# ----------------------------------------------------------------------
def _fetch_flask_version(self):
from flask import __version__ # pylint: disable=import-error,import-outside-toplevel
# pylint: disable-next=import-error,import-outside-toplevel,no-name-in-module
from flask import __version__
self._flask_version = __version__

# ----------------------------------------------------------------------
def _get_extra_fields(self, record):
from flask import request # pylint: disable=import-error,import-outside-toplevel
# pylint: disable-next=import-error,import-outside-toplevel
from flask import request

extra_fields = super()._get_extra_fields(record)
Schema = self.MessageSchema

extra_fields[Schema.FLASK_VERSION] = self._flask_version
if request: # request might be unbound in other threads
extra_fields[Schema.REQ_USER_AGENT] = str(request.user_agent) if request.user_agent else ''
extra_fields[Schema.REQ_USER_AGENT] = (str(request.user_agent)
if request.user_agent else '')
extra_fields[Schema.REQ_REMOTE_ADDRESS] = request.remote_addr
extra_fields[Schema.REQ_HOST] = request.host.split(':', 1)[0]
extra_fields[Schema.REQ_URI] = request.url
1 change: 1 addition & 0 deletions logstash_async/utils.py
Original file line number Diff line number Diff line change
@@ -64,6 +64,7 @@ def import_string(dotted_path):


# ----------------------------------------------------------------------
# pylint: disable-next=invalid-name
class normalize_ecs_dict:
"""
Convert dotted ecs fields into nested objects.
27 changes: 19 additions & 8 deletions tests/formatter_test.py
Original file line number Diff line number Diff line change
@@ -3,18 +3,25 @@
# This software may be modified and distributed under the terms
# of the MIT license. See the LICENSE file for details.

import socket
from contextlib import suppress
from logging import FileHandler, makeLogRecord
from types import SimpleNamespace
from unittest.mock import patch
import os
import socket
import sys
import unittest
from types import SimpleNamespace
from unittest.mock import patch

from logstash_async.formatter import (
DjangoLogstashEcsFormatter,
DjangoLogstashFormatter,
FlaskLogstashEcsFormatter,
FlaskLogstashFormatter,
LogstashEcsFormatter,
LogstashFormatter,
)
import logstash_async
from logstash_async.formatter import LogstashFormatter, DjangoLogstashFormatter, FlaskLogstashFormatter, \
LogstashEcsFormatter, DjangoLogstashEcsFormatter, FlaskLogstashEcsFormatter


# pylint: disable=protected-access

@@ -190,12 +197,15 @@ class _LogstashEcsFormatter(LogstashEcsFormatter):

class DjangoTestMixin:
@classmethod
def setUpClass(cls):
def setUpClass(cls): # pylint: disable=invalid-name
super().setUpClass()

import django
# pylint: disable=import-outside-toplevel
from django.conf import settings
from django.http import HttpRequest
import django

# pylint: enable=import-outside-toplevel

with suppress(RuntimeError):
settings.configure()
@@ -342,9 +352,10 @@ def test_default_schema(self):

class FlaskTestMixin:
@classmethod
def setUpClass(cls):
def setUpClass(cls): # pylint: disable=invalid-name
super().setUpClass()

# pylint: disable-next=import-outside-toplevel,no-name-in-module
from flask import __version__
cls.flask_version = __version__

3 changes: 2 additions & 1 deletion tests/utils_test.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import unittest
from copy import deepcopy
import unittest

from logstash_async.utils import normalize_ecs_dict

@@ -45,5 +45,6 @@ def test_normalization(self):

with self.subTest('source dict not mutated'):
self.assertDictEqual(d, d_copy)
# pylint: disable-next=unsubscriptable-object
result['c']['d'].append(22)
self.assertDictEqual(d, d_copy)