5.0.0

- Include MTA-STS and BIMI results in CSV output - Renamed `include_dmarc_tag_descriptions` parameter in `checkdmarc.check_domains()` to `include_tag_descriptions` - Added the `include_tag_descriptions` parameter to `checkdmarc.bimi.check_bimi()` - Ignore encoding value when checking the `Content-Type` header during the MTA-STS policy download - Added the exception class `MTASTSPolicyDownloadError` - Update documentation
domainaware · Dec 20, 2023 · 69386f6 · 69386f6
1 parent 655234b
commit 69386f6
Show file tree

Hide file tree

Showing 10 changed files with 143 additions and 57 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -41,6 +41,7 @@
         "maxdepth",
         "mcsv",
         "modindex",
+        "MTASTS",
         "NAMESERVER",
         "nojekyll",
         "nonauth",

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+5.0.1
+-----
+
+- Include MTA-STS and BIMI results in CSV output
+- Renamed `include_dmarc_tag_descriptions` parameter in `checkdmarc.check_domains()` to `include_tag_descriptions`
+- Added the `include_tag_descriptions` parameter to `checkdmarc.bimi.check_bimi()`
+- Ignore encoding value when checking the `Content-Type` header during the MTA-STS policy download
+- Added the exception class `MTASTSPolicyDownloadError`
+- Update documentation
+
 5.0.0
 -----
 

diff --git a/checkdmarc/__init__.py b/checkdmarc/__init__.py
@@ -45,7 +45,7 @@ def check_domains(domains: list[str], parked: bool = False,
                   approved_mx_hostnames: bool = None,
                   skip_tls: bool = False,
                   bimi_selector: str = None,
-                  include_dmarc_tag_descriptions: bool = False,
+                  include_tag_descriptions: bool = False,
                   nameservers: list[str] = None,
                   resolver: dns.resolver.Resolver = None,
                   timeout: float = 2.0,
@@ -61,7 +61,7 @@ def check_domains(domains: list[str], parked: bool = False,
         approved_mx_hostnames (list): A list of approved MX hostname
         skip_tls (bool): Skip STARTTLS testing
         bimi_selector (str): The BIMI selector to test
-        include_dmarc_tag_descriptions (bool): Include descriptions of DMARC
+        include_tag_descriptions (bool): Include descriptions of
                                                tags and/or tag values in the
                                                results
         nameservers (list): A list of nameservers to query
@@ -142,18 +142,20 @@ def check_domains(domains: list[str], parked: bool = False,
         domain_results["dmarc"] = check_dmarc(
             domain,
             parked=parked,
-            include_dmarc_tag_descriptions=include_dmarc_tag_descriptions,
+            include_dmarc_tag_descriptions=include_tag_descriptions,
             nameservers=nameservers,
             resolver=resolver,
             timeout=timeout
             )
 
         if bimi_selector is not None:
-            domain_results["bimi"] = check_bimi(domain,
-                                                selector=bimi_selector,
-                                                nameservers=nameservers,
-                                                resolver=resolver,
-                                                timeout=timeout)
+            domain_results["bimi"] = check_bimi(
+                domain,
+                selector=bimi_selector,
+                include_tag_descriptions=include_tag_descriptions,
+                nameservers=nameservers,
+                resolver=resolver,
+                timeout=timeout)
 
         results.append(domain_results)
         if wait > 0.0:
@@ -238,6 +240,7 @@ def results_to_csv_rows(results: Union[dict, list[dict]]) -> list[dict]:
         row = dict()
         ns = result["ns"]
         mx = result["mx"]
+        _mta_sts = result["mta_sts"]
         _spf = result["spf"]
         _dmarc = result["dmarc"]
         row["domain"] = result["domain"]
@@ -248,6 +251,24 @@ def results_to_csv_rows(results: Union[dict, list[dict]]) -> list[dict]:
             row["ns_error"] = ns["error"]
         else:
             row["ns_warnings"] = "|".join(ns["warnings"])
+        if "error" in _mta_sts:
+            row["mta_sts_error"] = _mta_sts["error"]
+        else:
+            row["mta_sts_id"] = _mta_sts["id"]
+            row["mta_sts_mode"] = _mta_sts["policy"]["mode"]
+            row["mta_sts_max_age"] = _mta_sts["policy"]["max_age"]
+            row["mta_sts_mx"] = "|".join(_mta_sts["policy"]["mx"])
+            row["mta_sts_warnings"] = "|".join(_mta_sts["warnings"])
+        if "bimi" in result:
+            _bimi = result["bimi"]
+            row["bimi_warnings"] = "|".join(_bimi["warnings"])
+            row["bimi_selector"] = _bimi["selector"]
+            if "error" in _bimi:
+                row["bimi_error"] = _bimi["error"]
+                if "l" in _bimi["tags"]:
+                    row["bimi_l"] = _bimi["tags"]["l"]["value"]
+                if "a" in _bimi["tags"]:
+                    row["bimi_a"] = _bimi["tags"]["a"]["value"]
         row["mx"] = "|".join(list(
             map(lambda r: f"{r['preference']}, {r['hostname']}", mx["hosts"])))
         tls = None
@@ -334,9 +355,10 @@ def results_to_csv(results: dict) -> str:
               "dmarc_adkim", "dmarc_aspf",
               "dmarc_fo", "dmarc_p", "dmarc_pct", "dmarc_rf", "dmarc_ri",
               "dmarc_rua", "dmarc_ruf", "dmarc_sp",
-              "mx", "tls", "starttls", "spf_record", "dmarc_record",
-              "dmarc_record_location", "mx_error",
-              "mx_warnings", "spf_error",
+              "tls", "starttls", "spf_record", "dmarc_record",
+              "dmarc_record_location", "mx", "mx_error", "mx_warnings",
+              "mta_sts_id", "mta_sts_mode", "mta_sts_max_age",
+              "mta_sts_mx", "mta_sts_error", "mta_sts_warnings", "spf_error",
               "spf_warnings", "dmarc_error", "dmarc_warnings",
               "ns", "ns_error", "ns_warnings"]
     output = StringIO(newline="\n")

diff --git a/checkdmarc/_cli.py b/checkdmarc/_cli.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-"""The command line interface"""
+"""TValidates and parses email-related DNS records"""
 
 from __future__ import annotations
 
@@ -41,7 +41,7 @@ def _main():
     arg_parser.add_argument("--mx", nargs="+",
                             help="approved MX hostname substrings")
     arg_parser.add_argument("-d", "--descriptions", action="store_true",
-                            help="include descriptions of DMARC tags in "
+                            help="include descriptions of tags in "
                                  "the JSON output")
     arg_parser.add_argument("-f", "--format", default="json",
                             help="specify JSON or CSV screen output format")
@@ -96,7 +96,7 @@ def _main():
                             parked=args.parked,
                             approved_nameservers=args.ns,
                             approved_mx_hostnames=args.mx,
-                            include_dmarc_tag_descriptions=args.descriptions,
+                            include_tag_descriptions=args.descriptions,
                             nameservers=args.nameserver, timeout=args.timeout,
                             bimi_selector=args.bimi_selector,
                             wait=args.wait)

diff --git a/checkdmarc/_constants.py b/checkdmarc/_constants.py
@@ -18,7 +18,7 @@
 See the License for the specific language governing permissions and
 limitations under the License."""
 
-__version__ = "5.0.0"
+__version__ = "5.0.1"
 
 OS = platform.system()
 OS_RELEASE = platform.release()

diff --git a/checkdmarc/bimi.py b/checkdmarc/bimi.py
@@ -352,6 +352,7 @@ def parse_bimi_record(
             raise InvalidBIMITag(f"{tag} is not a valid BIMI record tag")
         tags[tag] = OrderedDict(value=tag_value)
         if include_tag_descriptions:
+            tags[tag]["name"] = bimi_tags[tag]["name"]
             tags[tag]["description"] = bimi_tags[tag]["description"]
         if tag == "a" and tag_value != "":
             try:
@@ -372,6 +373,7 @@ def parse_bimi_record(
 
 
 def check_bimi(domain: str, selector: str = "default",
+               include_tag_descriptions: bool = False,
                nameservers: list[str] = None,
                resolver: dns.resolver.Resolver = None,
                timeout: float = 2.0) -> OrderedDict:
@@ -386,6 +388,7 @@ def check_bimi(domain: str, selector: str = "default",
     Args:
         domain (str): A domain name
         selector (str): The BIMI selector
+        include_tag_descriptions (bool): Include descriptions in parsed results
         nameservers (list): A list of nameservers to query
         resolver (dns.resolver.Resolver): A resolver object to use for DNS
                                           requests
@@ -415,7 +418,9 @@ def check_bimi(domain: str, selector: str = "default",
             timeout=timeout)
         bimi_results["selector"] = selector
         bimi_results["record"] = bimi_query["record"]
-        parsed_bimi = parse_bimi_record(bimi_results["record"])
+        parsed_bimi = parse_bimi_record(
+            bimi_results["record"],
+            include_tag_descriptions=include_tag_descriptions)
         bimi_results["tags"] = parsed_bimi["tags"]
         bimi_results["warnings"] = parsed_bimi["warnings"]
     except BIMIError as error:

diff --git a/checkdmarc/mta_sts.py b/checkdmarc/mta_sts.py
@@ -41,10 +41,6 @@
 MTA_STS_MX_REGEX = re.compile(MTA_STS_MX_REGEX_STRING, re.IGNORECASE)
 
 
-class _MTASTSWarning(Exception):
-    """Raised when a non-fatal MTA-STS error occurs"""
-
-
 class MTASTSError(Exception):
     """Raised when a fatal MTA-STS error occurs"""
     def __init__(self, msg: str, data: dict = None):
@@ -96,7 +92,11 @@ class MultipleMTASTSRecords(MTASTSError):
 
 
 class MTASTSPolicyError(MTASTSError):
-    """Raised when the MTA-STS policy cannot be obtained or parsed"""
+    """Raised when the MTA-STS policy cannot be downloaded or parsed"""
+
+
+class MTASTSPolicyDownloadError(MTASTSPolicyError):
+    """Raised when the MTA-STS policy cannot be downloaded"""
 
 
 class MTASTSPolicySyntaxError(MTASTSPolicyError):
@@ -158,9 +158,9 @@ def query_mta_sts_record(domain: str,
         :exc:`checkdmarc.mta_sts.MultipleMTASTSRecords`
 
     """
+    domain = domain.lower()
     logging.debug(f"Checking for a MTA-STS record on {domain}")
     warnings = []
-    domain = domain.lower()
     target = f"_mta-sts.{domain}"
     sts_record = None
     sts_record_count = 0
@@ -296,7 +296,7 @@ def download_mta_sts_policy(domain: str) -> OrderedDict:
                      - ``warnings`` - A list of any warning conditions found
 
     Raises:
-        :exc:`checkdmarc.mta_sts.MTASTSPolicyError`
+        :exc:`checkdmarc.mta_sts.MTASTSPolicyDownloadError`
     """
     warnings = []
     headers = {"User-Agent": USER_AGENT}
@@ -309,7 +309,8 @@ def download_mta_sts_policy(domain: str) -> OrderedDict:
         response = session.get(url)
         response.raise_for_status()
         if "Content-Type" in response.headers:
-            content_type = response.headers["Content-Type"]
+            content_type = response.headers["Content-Type"].split(";")[0]
+            content_type = content_type.strip()
             if content_type != expected_content_type:
                 warnings.append(f"Content-Type header should be "
                                 f"{expected_content_type} not {content_type}")
@@ -318,7 +319,7 @@ def download_mta_sts_policy(domain: str) -> OrderedDict:
                             f"be set to {expected_content_type}")
 
     except Exception as e:
-        raise MTASTSPolicyError(str(e))
+        raise MTASTSPolicyDownloadError(str(e))
 
     return OrderedDict(policy=response.text, warnings=warnings)
 

diff --git a/docs/source/api.md b/docs/source/api.md
@@ -1,7 +1,57 @@
 # API
 
+## checkdmarc
+
 ```{eval-rst}
 .. automodule:: checkdmarc
    :members:
-   :undoc-members:
+```
+
+## checkdmarc.bimi
+
+```{eval-rst}
+.. automodule:: checkdmarc.bimi
+   :members:
+```
+
+## checkdmarc.dmarc
+
+```{eval-rst}
+.. automodule:: checkdmarc.dmarc
+   :members:
+```
+
+## checkdmarc.dnssec
+
+```{eval-rst}
+.. automodule:: checkdmarc.dnssec
+   :members:
+```
+
+## checkdmarc.mta_sts
+
+```{eval-rst}
+.. automodule:: checkdmarc.mta_sts
+   :members:
+```
+
+## checkdmarc.smtp
+
+```{eval-rst}
+.. automodule:: checkdmarc.smtp
+   :members:
+```
+
+## checkdmarc.spf
+
+```{eval-rst}
+.. automodule:: checkdmarc.spf
+   :members:
+```
+
+## checkdmarc.utils
+
+```{eval-rst}
+.. automodule:: checkdmarc.utils
+   :members:
 ```
diff --git a/docs/source/cli.md b/docs/source/cli.md
@@ -1,39 +1,36 @@
 # CLI
 
 ```text
-usage: checkdmarc  [-h] [-p] [--ns NS [NS ...]] [--mx MX [MX ...]] [-d]
-                    [-f FORMAT] [-o OUTPUT [OUTPUT ...]]
-                    [-n NAMESERVER [NAMESERVER ...]] [-t TIMEOUT] [-v]
-                    [-w WAIT] [--skip-tls] [--debug]
-                    domain [domain ...]
+usage: checkdmarc [-h] [-p] [--ns NS [NS ...]] [--mx MX [MX ...]] [-d] [-f FORMAT] [-o OUTPUT [OUTPUT ...]]
+                  [-n NAMESERVER [NAMESERVER ...]] [-t TIMEOUT] [-b BIMI_SELECTOR] [-v] [-w WAIT] [--skip-tls]
+                  [--debug]
+                  domain [domain ...]
 
-  Validates and parses SPF amd DMARC DNS records
+Validates and parses email-related DNS records
 
-  positional arguments:
-    domain                one or more domains, or a single path to a file
-                          containing a list of domains
+positional arguments:
+  domain                one or more domains, or a single path to a file containing a list of domains
 
-  optional arguments:
-    -h, --help            show this help message and exit
-    -p, --parked          indicate that the domains are parked
-    --ns NS [NS ...]      approved nameserver substrings
-    --mx MX [MX ...]      approved MX hostname substrings
-    -d, --descriptions    include descriptions of DMARC tags in the JSON output
-    -f FORMAT, --format FORMAT
-                          specify JSON or CSV screen output format
-    -o OUTPUT [OUTPUT ...], --output OUTPUT [OUTPUT ...]
-                          one or more file paths to output to (must end in .json
-                          or .csv) (silences screen output)
-    -n NAMESERVER [NAMESERVER ...], --nameserver NAMESERVER [NAMESERVER ...]
-                          nameservers to query (Default is Cloudflare's
-    -t TIMEOUT, --timeout TIMEOUT
-                          number of seconds to wait for an answer from DNS
-                          (default 2.0)
-    -v, --version         show program's version number and exit
-    -w WAIT, --wait WAIT  number of seconds to wait between checking domains
-                          (default 0.0)
-    --skip-tls            skip TLS/SSL testing
-    --debug               enable debugging output
+options:
+  -h, --help            show this help message and exit
+  -p, --parked          indicate that the domains are parked
+  --ns NS [NS ...]      approved nameserver substrings
+  --mx MX [MX ...]      approved MX hostname substrings
+  -d, --descriptions    include descriptions of tags in the JSON output
+  -f FORMAT, --format FORMAT
+                        specify JSON or CSV screen output format
+  -o OUTPUT [OUTPUT ...], --output OUTPUT [OUTPUT ...]
+                        one or more file paths to output to (must end in .json or .csv) (silences screen output)
+  -n NAMESERVER [NAMESERVER ...], --nameserver NAMESERVER [NAMESERVER ...]
+                        nameservers to query
+  -t TIMEOUT, --timeout TIMEOUT
+                        number of seconds to wait for an answer from DNS (default 2.0)
+  -b BIMI_SELECTOR, --bimi-selector BIMI_SELECTOR
+                        Check for a BIMI record at the provided selector
+  -v, --version         show program's version number and exit
+  -w WAIT, --wait WAIT  number of seconds to wait between checking domains (default 0.0)
+  --skip-tls            skip TLS/SSL testing
+  --debug               enable debugging output
 ```
 
 ## Example

diff --git a/docs/source/index.md b/docs/source/index.md
@@ -20,7 +20,7 @@
 
 ```{toctree}
 ---
-maxdepth: 2
+maxdepth: 1
 ---
 installation
 api