feat(breadcrumbs): add _meta-tags for truncation of breadcrumbs

sentry_sdk/scope.py

@@ -29,6 +29,7 @@
     Transaction,
 )
 from sentry_sdk.utils import (
+    AnnotatedDeque,
     capture_internal_exception,
     capture_internal_exceptions,
     ContextVar,
@@ -181,6 +182,7 @@ class Scope:
         "_contexts",
         "_extras",
         "_breadcrumbs",
+        "_n_breadcrumbs_truncated",
         "_event_processors",
         "_error_processors",
         "_should_capture",
@@ -205,6 +207,7 @@ def __init__(self, ty=None, client=None):
 
         self._name = None  # type: Optional[str]
         self._propagation_context = None  # type: Optional[PropagationContext]
+        self._n_breadcrumbs_truncated = 0  # type: int
 
         self.client = NonRecordingClient()  # type: sentry_sdk.client.BaseClient
 
@@ -238,6 +241,7 @@ def __copy__(self):
         rv._extras = dict(self._extras)
 
         rv._breadcrumbs = copy(self._breadcrumbs)
+        rv._n_breadcrumbs_truncated = copy(self._n_breadcrumbs_truncated)
         rv._event_processors = list(self._event_processors)
         rv._error_processors = list(self._error_processors)
         rv._propagation_context = self._propagation_context
@@ -906,6 +910,7 @@ def clear_breadcrumbs(self):
         # type: () -> None
         """Clears breadcrumb buffer."""
         self._breadcrumbs = deque()  # type: Deque[Breadcrumb]
+        self._n_breadcrumbs_truncated = 0
 
     def add_attachment(
         self,
@@ -973,6 +978,7 @@ def add_breadcrumb(self, crumb=None, hint=None, **kwargs):
 
         while len(self._breadcrumbs) > max_breadcrumbs:
             self._breadcrumbs.popleft()
+            self._n_breadcrumbs_truncated += 1
 
     def start_transaction(
         self,
@@ -1339,6 +1345,15 @@ def _apply_breadcrumbs_to_event(self, event, hint, options):
             logger.debug("Error when sorting breadcrumbs", exc_info=err)
             pass
 
+        # Add annotation that breadcrumbs were truncated
+        if self._n_breadcrumbs_truncated:
+            original_length = (
+                len(event["breadcrumbs"]["values"]) + self._n_breadcrumbs_truncated
+            )
+            event["breadcrumbs"]["values"] = AnnotatedDeque.truncated_breadcrumbs(
+                event["breadcrumbs"]["values"], original_length
+            )
+
     def _apply_user_to_event(self, event, hint, options):
         # type: (Event, Hint, Optional[Dict[str, Any]]) -> None
         if event.get("user") is None and self._user is not None:

sentry_sdk/scrubber.py

@@ -146,9 +146,14 @@ def scrub_breadcrumbs(self, event):
         with capture_internal_exceptions():
             if "breadcrumbs" in event:
                 if "values" in event["breadcrumbs"]:
-                    for value in event["breadcrumbs"]["values"]:
-                        if "data" in value:
-                            self.scrub_dict(value["data"])
+                    if isinstance(event["breadcrumbs"]["values"], AnnotatedValue):
+                        for value in event["breadcrumbs"]["values"].value:
+                            if "data" in value:
+                                self.scrub_dict(value["data"])
+                    else:
+                        for value in event["breadcrumbs"]["values"]:
+                            if "data" in value:
+                                self.scrub_dict(value["data"])
 
     def scrub_frames(self, event):
         # type: (Event) -> None

sentry_sdk/utils.py

@@ -42,6 +42,7 @@
         Callable,
         cast,
         ContextManager,
+        Deque,
         Dict,
         Iterator,
         List,
@@ -475,6 +476,52 @@ def substituted_because_contains_sensitive_data(cls):
         )
 
 
+class AnnotatedDeque(AnnotatedValue):
+    """
+    Meta information for a data field in the event payload.
+    This is to tell Relay that we have tampered with the fields value.
+    See:
+    https://github.com/getsentry/relay/blob/be12cd49a0f06ea932ed9b9f93a655de5d6ad6d1/relay-general/src/types/meta.rs#L407-L423
+    """
+
+    __slots__ = ("value", "metadata")
+
+    def __init__(self, value, metadata):
+        # type: (Deque[Any], Dict[str, Any]) -> None
+        self.value = value
+        self.metadata = metadata
+
+    def __eq__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, AnnotatedValue):
+            return False
+
+        return self.value == other.value and self.metadata == other.metadata
+
+    def append(self, other):
+        # type: (Any) -> None
+        self.value.append(other)
+
+    def extend(self, other):
+        # type: (Any) -> None
+        self.value.extend(other)
+
+    def popleft(self):
+        self.value.popleft()
+
+    def __len__(self):
+        return len(self.value)
+
+    @classmethod
+    def truncated(cls, value, n_truncated):
+        # type: (Deque[Any], int) -> AnnotatedValue
+        """Data was removed because the number of elements exceeded the maximum limit."""
+        return AnnotatedDeque(
+            value=value,
+            metadata={"len": [n_truncated]},  # Remark
+        )
+
+
 if TYPE_CHECKING:
     from typing import TypeVar
 

tests/test_client.py

@@ -1058,7 +1058,6 @@ def test_max_breadcrumbs_option(
         add_breadcrumb({"type": "sourdough"})
 
     capture_message("dogs are great")
-
     assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs
 
 

tests/test_scrubber.py

@@ -137,7 +137,10 @@ def test_breadcrumb_extra_scrubbing(sentry_init, capture_events):
 
     assert event["_meta"]["extra"]["auth"] == {"": {"rem": [["!config", "s"]]}}
     assert event["_meta"]["breadcrumbs"] == {
-        "values": {"0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}}}
+        "values": {
+            "": {"len": [1]},
+            "0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}},
+        }
     }