Improve git url detection for generic purls

Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Mar 20, 2024 · ef9a62e · ef9a62e
1 parent 0f6543d
commit ef9a62e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 6 deletions.
diff --git a/contrib/cpe_research.py b/contrib/cpe_research.py
@@ -27,7 +27,7 @@ def get_cve_data(db_conn, index_hits: list[dict, Any]):
         db_conn, _ = db6.get(read_only=True)
     for ahit in index_hits:
         results: apsw.Cursor = db_conn.execute(
-            "SELECT json_object('source', source_data) FROM cve_data WHERE cve_id = ? AND type = ? ORDER BY cve_id DESC;",
+            "SELECT distinct json_object('source', source_data) FROM cve_data WHERE cve_id = ? AND type = ? ORDER BY cve_id DESC;",
             (ahit[0], ahit[1]),
         )
         for res in results:

diff --git a/vdb/lib/nvd.py b/vdb/lib/nvd.py
@@ -58,7 +58,9 @@ def filterable_git_url(url: str) -> bool:
         "vulnerabilities",
         "about.gitlab.com",
         "xss",
-        "lists.apache.org"
+        "lists.apache.org",
+        "cisagov",
+        "nu11secur1ty"
     ):
         if part in url.lower():
             return True
@@ -306,7 +308,6 @@ def convert_api_vuln_detail(vuln: dict) -> list[VulnerabilityDetail] | None:
                 # Try to extract any git references from related urls
                 # See: https://github.com/AppThreat/vulnerability-db/issues/91
                 for agit_url in git_urls:
-                    git_repo_name = None
                     # Ignore obvious filterable urls
                     if filterable_git_url(agit_url):
                         continue
@@ -326,9 +327,7 @@ def convert_api_vuln_detail(vuln: dict) -> list[VulnerabilityDetail] | None:
                         ):
                             if part in agit_url:
                                 git_repo_name = agit_url.split(part)[0]
-                    if git_repo_name and not parsed_git_repo_names.get(
-                        git_repo_name
-                    ):
+                    if not parsed_git_repo_names.get(git_repo_name):
                         parsed_git_repo_names[git_repo_name] = True
                         git_repo_name = (
                             git_repo_name.removeprefix("https://")