diff --git a/mir_eval/display.py b/mir_eval/display.py
index 8a276845..f493cddc 100644
--- a/mir_eval/display.py
+++ b/mir_eval/display.py
@@ -145,6 +145,9 @@ def segments(
 
     if new_axes:
         ax.set_ylim([0, 1])
+        ax.set_yticks([])
+        # Very small positive number here to preserve ticks
+        ax.margins(x=1e-3, y=0, tight=None)
 
     # Infer height
     if base is None:
@@ -162,7 +165,7 @@ def segments(
             continue
 
         # style = next(cycler)
-        _bar = ax.bar([0], [0], visible=False)
+        _bar = ax.bar([0.5], [0.5], visible=False)
         style = {
             k: v
             for k, v in _bar[0].properties().items()
@@ -191,14 +194,6 @@ def segments(
             )
             ann.set_clip_path(rect)
 
-    if new_axes:
-        ax.set_yticks([])
-        ax.set_xlim(intervals.min(), intervals.max())
-
-    # Only expand if we have data
-    if intervals.size:
-        __expand_limits(ax, [intervals.min(), intervals.max()], which="x")
-
     return ax
 
 
@@ -342,6 +337,10 @@ def labeled_intervals(
         ax.set_yticklabels(ticks, va="bottom")
         ax.yaxis.set_major_formatter(IntervalFormatter(base, ticks))
 
+    if new_axes:
+        # Very small positive number here to preserve ticks
+        ax.margins(x=1e-3, y=0, tight=None)
+
     if base.size:
         __expand_limits(ax, [base.min(), (base + height).max()], which="y")
     if intervals.size:
diff --git a/tests/baseline_images/test_display/test_display_labeled_intervals.png b/tests/baseline_images/test_display/test_display_labeled_intervals.png
index a0418c8b..de00964a 100644
Binary files a/tests/baseline_images/test_display/test_display_labeled_intervals.png and b/tests/baseline_images/test_display/test_display_labeled_intervals.png differ
diff --git a/tests/baseline_images/test_display/test_display_labeled_intervals_compare_noextend.png b/tests/baseline_images/test_display/test_display_labeled_intervals_compare_noextend.png
index a0a2bc05..f595c103 100644
Binary files a/tests/baseline_images/test_display/test_display_labeled_intervals_compare_noextend.png and b/tests/baseline_images/test_display/test_display_labeled_intervals_compare_noextend.png differ
diff --git a/tests/baseline_images/test_display/test_display_segment.png b/tests/baseline_images/test_display/test_display_segment.png
index a0cb5d35..acdd967c 100644
Binary files a/tests/baseline_images/test_display/test_display_segment.png and b/tests/baseline_images/test_display/test_display_segment.png differ
diff --git a/tests/baseline_images/test_display/test_display_segment_text.png b/tests/baseline_images/test_display/test_display_segment_text.png
index 79efe951..1923ba73 100644
Binary files a/tests/baseline_images/test_display/test_display_segment_text.png and b/tests/baseline_images/test_display/test_display_segment_text.png differ
diff --git a/tests/baseline_images/test_display/test_display_separation.png b/tests/baseline_images/test_display/test_display_separation.png
index 26e61999..2a1cade4 100644
Binary files a/tests/baseline_images/test_display/test_display_separation.png and b/tests/baseline_images/test_display/test_display_separation.png differ
diff --git a/tests/baseline_images/test_display/test_display_separation_label.png b/tests/baseline_images/test_display/test_display_separation_label.png
index 70513020..cdb0056f 100644
Binary files a/tests/baseline_images/test_display/test_display_separation_label.png and b/tests/baseline_images/test_display/test_display_separation_label.png differ