Add re.findall to pick out re matches

Signed-off-by: James Hewitt <[email protected]>

docs/source/filters.rst

@@ -77,6 +77,7 @@ At the moment, the following filters are built-in:
 - **ical2text**: Convert `iCalendar`_ to plaintext
 - **ocr**: Convert text in images to plaintext using Tesseract OCR
 - **re.sub**: Replace text with regular expressions using Python's re.sub
+- **re.findall**: Find all non-overlapping matches using Python's re.findall
 - **reverse**: Reverse input items
 - **sha1sum**: Calculate the SHA-1 checksum of the content
 - **shellpipe**: Filter using a shell command

lib/urlwatch/filters.py

@@ -848,6 +848,26 @@ def filter(self, data, subfilter):
         return re.sub(subfilter['pattern'], subfilter.get('repl', ''), data)
 
 
+class RegexFindall(FilterBase):
+    """Pick out regular expressions using Python's re.findall"""
+
+    __kind__ = 're.findall'
+
+    __supported_subfilters__ = {
+        'pattern': 'Regular expression to search for (required)',
+        'repl': 'Replacement string (default: empty string)',
+    }
+
+    __default_subfilter__ = 'pattern'
+
+    def filter(self, data, subfilter):
+        if 'pattern' not in subfilter:
+            raise ValueError('{} needs a pattern'.format(self.__kind__))
+
+        # Default: Replace with empty string if no "repl" value is set
+        return "\n".join([match.expand(subfilter.get('repl', '\\g<0>')) for match in re.finditer(subfilter['pattern'], data)])
+
+
 class SortFilter(FilterBase):
     """Sort input items"""
 

lib/urlwatch/tests/data/filter_tests.yaml

@@ -326,6 +326,32 @@ re_sub_multiline:
     One Line
     
     Another Line
+re_findall:
+  filter:
+    - re.findall: '-[a-z][a-z][a-z]-'
+  data: |-
+    Some-abc-things-def-on-ghi-this-line-and
+    some-jkl-more-mno-here
+  expected_result: |-
+    -abc-
+    -def-
+    -ghi-
+    -jkl-
+    -mno-
+re_findall_repl:
+  filter:
+    - re.findall:
+        pattern: '-([a-z])([a-z])([a-z])-'
+        repl: '\3\2\1'
+  data: |-
+    Some-abc-things-def-on-ghi-this-line-and
+    some-jkl-more-mno-here
+  expected_result: |-
+    cba
+    fed
+    ihg
+    lkj
+    onm
 strip:
   filter: strip
   data: "  The rose is red;   \n\nthe violet's blue.\nSugar is sweet,       \nand so are you.   "