OCTO-10874-ascii-duplicates

pycaption/scc/specialized_collections.py

@@ -8,7 +8,7 @@
 )
 from .constants import (
     PAC_BYTES_TO_POSITIONING_MAP, COMMANDS, PAC_TAB_OFFSET_COMMANDS,
-    MICROSECONDS_PER_CODEWORD,
+    MICROSECONDS_PER_CODEWORD, EXTENDED_CHARS, CHARACTERS
 )
 
 PopOnCue = collections.namedtuple("PopOnCue", "buffer, start, end")
@@ -423,11 +423,11 @@ def remove_ascii_duplicate(self, accented_character):
 
         :type accented_character: str
         """
-        if self._collection and self._collection[-1].is_text_node() and \
-                self._collection[-1].text:
-            ascii_char = unicodedata.normalize('NFD', accented_character)\
-                .encode('ascii', 'ignore').decode("utf-8")
-            if ascii_char and self._collection[-1].text[-1] == ascii_char:
+        is_text_node = self._collection and self._collection[-1].is_text_node() and self._collection[-1].text
+        if is_text_node:
+            last_char = self._collection[-1].text[-1]
+            is_char = last_char in CHARACTERS.values() or last_char in EXTENDED_CHARS.values()
+            if is_char:
                 self._collection[-1].text = self._collection[-1].text[:-1]
 
 

tests/test_scc.py

@@ -263,7 +263,8 @@ def test_freeze_rollup_captions_contents(self, sample_scc_roll_up_ru2):
             'WE SERVE.',
             '®°½',
             'ABû',
-            'ÁÁÉÓ¡',
+            # 'ÁÁÉÓ¡',
+            '¡',
             "WHERE YOU'RE STANDING NOW,",
             "LOOKING OUT THERE, THAT'S AL",
             'THE CROWD.',
@@ -272,7 +273,6 @@ def test_freeze_rollup_captions_contents(self, sample_scc_roll_up_ru2):
             'And wildlife.',
             '>> Bike Iowa, your source for',
         ]
-
         assert expected_texts == actual_texts
 
     def test_multiple_formats(self, sample_scc_multiple_formats):