From 78ff303fb0e590a8120dd82d34e25126089367b8 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 29 Jan 2025 16:13:41 +0100 Subject: [PATCH 1/4] feat: add _meta annotations for breadcrumb truncataion --- sentry_sdk/scope.py | 12 ++++++++++++ sentry_sdk/scrubber.py | 11 ++++++++--- sentry_sdk/utils.py | 11 ++++++++++- tests/test_client.py | 8 +++++--- tests/test_scrubber.py | 5 ++++- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py index c22cdfb030..19a2a3c602 100644 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -29,6 +29,7 @@ Transaction, ) from sentry_sdk.utils import ( + AnnotatedValue, capture_internal_exception, capture_internal_exceptions, ContextVar, @@ -181,6 +182,7 @@ class Scope: "_contexts", "_extras", "_breadcrumbs", + "_breadcrumb_info", "_event_processors", "_error_processors", "_should_capture", @@ -205,6 +207,7 @@ def __init__(self, ty=None, client=None): self._name = None # type: Optional[str] self._propagation_context = None # type: Optional[PropagationContext] + self._breadcrumb_info = 0 # type: int self.client = NonRecordingClient() # type: sentry_sdk.client.BaseClient @@ -238,6 +241,7 @@ def __copy__(self): rv._extras = dict(self._extras) rv._breadcrumbs = copy(self._breadcrumbs) + rv._breadcrumb_info = copy(self._breadcrumb_info) rv._event_processors = list(self._event_processors) rv._error_processors = list(self._error_processors) rv._propagation_context = self._propagation_context @@ -906,6 +910,7 @@ def clear_breadcrumbs(self): # type: () -> None """Clears breadcrumb buffer.""" self._breadcrumbs = deque() # type: Deque[Breadcrumb] + self._breadcrumb_info = 0 def add_attachment( self, @@ -973,6 +978,7 @@ def add_breadcrumb(self, crumb=None, hint=None, **kwargs): while len(self._breadcrumbs) > max_breadcrumbs: self._breadcrumbs.popleft() + self._breadcrumb_info += 1 def start_transaction( self, @@ -1339,6 +1345,12 @@ def _apply_breadcrumbs_to_event(self, event, hint, options): logger.debug("Error when sorting breadcrumbs", exc_info=err) pass + # Add annotation that breadcrumbs were truncated + original_length = len(event["breadcrumbs"]["values"]) + self._breadcrumb_info + event["breadcrumbs"]["values"] = AnnotatedValue.truncated_breadcrumbs( + event["breadcrumbs"]["values"], original_length + ) + def _apply_user_to_event(self, event, hint, options): # type: (Event, Hint, Optional[Dict[str, Any]]) -> None if event.get("user") is None and self._user is not None: diff --git a/sentry_sdk/scrubber.py b/sentry_sdk/scrubber.py index f4755ea93b..6ee572ee57 100644 --- a/sentry_sdk/scrubber.py +++ b/sentry_sdk/scrubber.py @@ -146,9 +146,14 @@ def scrub_breadcrumbs(self, event): with capture_internal_exceptions(): if "breadcrumbs" in event: if "values" in event["breadcrumbs"]: - for value in event["breadcrumbs"]["values"]: - if "data" in value: - self.scrub_dict(value["data"]) + if isinstance(event["breadcrumbs"]["values"], AnnotatedValue): + for value in event["breadcrumbs"]["values"].value: + if "data" in value: + self.scrub_dict(value["data"]) + else: + for value in event["breadcrumbs"]["values"]: + if "data" in value: + self.scrub_dict(value["data"]) def scrub_frames(self, event): # type: (Event) -> None diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index 0fead48377..5a3f498573 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -58,7 +58,7 @@ from gevent.hub import Hub - from sentry_sdk._types import Event, ExcInfo + from sentry_sdk._types import Breadcrumb, Event, ExcInfo P = ParamSpec("P") R = TypeVar("R") @@ -474,6 +474,15 @@ def substituted_because_contains_sensitive_data(cls): }, ) + @classmethod + def truncated_breadcrumbs(cls, breadcrumbs, n_truncated): + # type: (list[Breadcrumb], int) -> AnnotatedValue + """Breadcrumbs were removed because the number of breadcrumbs exceeded their maximum limit.""" + return AnnotatedValue( + value=breadcrumbs, + metadata={"len": [n_truncated]}, # Remark + ) + if TYPE_CHECKING: from typing import TypeVar diff --git a/tests/test_client.py b/tests/test_client.py index 67f53d989a..6af43cfe69 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -22,7 +22,7 @@ set_tag, ) from sentry_sdk.spotlight import DEFAULT_SPOTLIGHT_URL -from sentry_sdk.utils import capture_internal_exception +from sentry_sdk.utils import AnnotatedValue, capture_internal_exception from sentry_sdk.integrations.executing import ExecutingIntegration from sentry_sdk.transport import Transport from sentry_sdk.serializer import MAX_DATABAG_BREADTH @@ -1058,8 +1058,10 @@ def test_max_breadcrumbs_option( add_breadcrumb({"type": "sourdough"}) capture_message("dogs are great") - - assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs + if isinstance(events[0]["breadcrumbs"]["values"], AnnotatedValue): + assert len(events[0]["breadcrumbs"]["values"].value) == expected_breadcrumbs + else: + assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs def test_multiple_positional_args(sentry_init): diff --git a/tests/test_scrubber.py b/tests/test_scrubber.py index 2c462153dd..9999d55d95 100644 --- a/tests/test_scrubber.py +++ b/tests/test_scrubber.py @@ -137,7 +137,10 @@ def test_breadcrumb_extra_scrubbing(sentry_init, capture_events): assert event["_meta"]["extra"]["auth"] == {"": {"rem": [["!config", "s"]]}} assert event["_meta"]["breadcrumbs"] == { - "values": {"0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}}} + "values": { + "": {"len": [1]}, + "0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}}, + } } From dceab057465adbc34eda80cae1e5eccb558d51e6 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 29 Jan 2025 16:15:31 +0100 Subject: [PATCH 2/4] rename var --- sentry_sdk/scope.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py index 19a2a3c602..e831c7b5f7 100644 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -182,7 +182,7 @@ class Scope: "_contexts", "_extras", "_breadcrumbs", - "_breadcrumb_info", + "_n_breadcrumbs_truncated", "_event_processors", "_error_processors", "_should_capture", @@ -207,7 +207,7 @@ def __init__(self, ty=None, client=None): self._name = None # type: Optional[str] self._propagation_context = None # type: Optional[PropagationContext] - self._breadcrumb_info = 0 # type: int + self._n_breadcrumbs_truncated = 0 # type: int self.client = NonRecordingClient() # type: sentry_sdk.client.BaseClient @@ -241,7 +241,7 @@ def __copy__(self): rv._extras = dict(self._extras) rv._breadcrumbs = copy(self._breadcrumbs) - rv._breadcrumb_info = copy(self._breadcrumb_info) + rv._n_breadcrumbs_truncated = copy(self._n_breadcrumbs_truncated) rv._event_processors = list(self._event_processors) rv._error_processors = list(self._error_processors) rv._propagation_context = self._propagation_context @@ -910,7 +910,7 @@ def clear_breadcrumbs(self): # type: () -> None """Clears breadcrumb buffer.""" self._breadcrumbs = deque() # type: Deque[Breadcrumb] - self._breadcrumb_info = 0 + self._n_breadcrumbs_truncated = 0 def add_attachment( self, @@ -978,7 +978,7 @@ def add_breadcrumb(self, crumb=None, hint=None, **kwargs): while len(self._breadcrumbs) > max_breadcrumbs: self._breadcrumbs.popleft() - self._breadcrumb_info += 1 + self._n_breadcrumbs_truncated += 1 def start_transaction( self, @@ -1346,7 +1346,9 @@ def _apply_breadcrumbs_to_event(self, event, hint, options): pass # Add annotation that breadcrumbs were truncated - original_length = len(event["breadcrumbs"]["values"]) + self._breadcrumb_info + original_length = ( + len(event["breadcrumbs"]["values"]) + self._n_breadcrumbs_truncated + ) event["breadcrumbs"]["values"] = AnnotatedValue.truncated_breadcrumbs( event["breadcrumbs"]["values"], original_length ) From 4e6bb41dad42fcf0c7e66243525ad4445e5f6bc6 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Fri, 31 Jan 2025 13:59:37 +0100 Subject: [PATCH 3/4] add AnnotatedDeque for better handling --- sentry_sdk/scope.py | 15 +++++++------- sentry_sdk/utils.py | 50 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py index e831c7b5f7..0bae626be8 100644 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -29,7 +29,7 @@ Transaction, ) from sentry_sdk.utils import ( - AnnotatedValue, + AnnotatedDeque, capture_internal_exception, capture_internal_exceptions, ContextVar, @@ -1346,12 +1346,13 @@ def _apply_breadcrumbs_to_event(self, event, hint, options): pass # Add annotation that breadcrumbs were truncated - original_length = ( - len(event["breadcrumbs"]["values"]) + self._n_breadcrumbs_truncated - ) - event["breadcrumbs"]["values"] = AnnotatedValue.truncated_breadcrumbs( - event["breadcrumbs"]["values"], original_length - ) + if self._n_breadcrumbs_truncated: + original_length = ( + len(event["breadcrumbs"]["values"]) + self._n_breadcrumbs_truncated + ) + event["breadcrumbs"]["values"] = AnnotatedDeque.truncated_breadcrumbs( + event["breadcrumbs"]["values"], original_length + ) def _apply_user_to_event(self, event, hint, options): # type: (Event, Hint, Optional[Dict[str, Any]]) -> None diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py index 5a3f498573..f87a602a08 100644 --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -42,6 +42,7 @@ Callable, cast, ContextManager, + Deque, Dict, Iterator, List, @@ -58,7 +59,7 @@ from gevent.hub import Hub - from sentry_sdk._types import Breadcrumb, Event, ExcInfo + from sentry_sdk._types import Event, ExcInfo P = ParamSpec("P") R = TypeVar("R") @@ -474,12 +475,49 @@ def substituted_because_contains_sensitive_data(cls): }, ) + +class AnnotatedDeque(AnnotatedValue): + """ + Meta information for a data field in the event payload. + This is to tell Relay that we have tampered with the fields value. + See: + https://github.com/getsentry/relay/blob/be12cd49a0f06ea932ed9b9f93a655de5d6ad6d1/relay-general/src/types/meta.rs#L407-L423 + """ + + __slots__ = ("value", "metadata") + + def __init__(self, value, metadata): + # type: (Deque[Any], Dict[str, Any]) -> None + self.value = value + self.metadata = metadata + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, AnnotatedValue): + return False + + return self.value == other.value and self.metadata == other.metadata + + def append(self, other): + # type: (Any) -> None + self.value.append(other) + + def extend(self, other): + # type: (Any) -> None + self.value.extend(other) + + def popleft(self): + self.value.popleft() + + def __len__(self): + return len(self.value) + @classmethod - def truncated_breadcrumbs(cls, breadcrumbs, n_truncated): - # type: (list[Breadcrumb], int) -> AnnotatedValue - """Breadcrumbs were removed because the number of breadcrumbs exceeded their maximum limit.""" - return AnnotatedValue( - value=breadcrumbs, + def truncated(cls, value, n_truncated): + # type: (Deque[Any], int) -> AnnotatedValue + """Data was removed because the number of elements exceeded the maximum limit.""" + return AnnotatedDeque( + value=value, metadata={"len": [n_truncated]}, # Remark ) From f9d006307d194b3e2acd7ca821847cfd9d6f0256 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Fri, 31 Jan 2025 14:02:23 +0100 Subject: [PATCH 4/4] remove special case for test --- tests/test_client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 6af43cfe69..05632479df 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -22,7 +22,7 @@ set_tag, ) from sentry_sdk.spotlight import DEFAULT_SPOTLIGHT_URL -from sentry_sdk.utils import AnnotatedValue, capture_internal_exception +from sentry_sdk.utils import capture_internal_exception from sentry_sdk.integrations.executing import ExecutingIntegration from sentry_sdk.transport import Transport from sentry_sdk.serializer import MAX_DATABAG_BREADTH @@ -1058,10 +1058,7 @@ def test_max_breadcrumbs_option( add_breadcrumb({"type": "sourdough"}) capture_message("dogs are great") - if isinstance(events[0]["breadcrumbs"]["values"], AnnotatedValue): - assert len(events[0]["breadcrumbs"]["values"].value) == expected_breadcrumbs - else: - assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs + assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs def test_multiple_positional_args(sentry_init):