Refactored and moved url to purl conversion to utils

Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Mar 23, 2024 · 1d08098 · 1d08098
1 parent d1ff802
commit 1d08098
Show file tree

Hide file tree

Showing 11 changed files with 726 additions and 289 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,19 @@
 
 This repo is a vulnerability database and package search for sources such as AppThreat vuln-list, OSV, NVD, and GitHub. Vulnerability data are downloaded from the sources and stored in a sqlite based storage with indexes to allow offline access and efficient searches.
 
+## Why vulnerability db?
+
+A good vulnerability database must have the following properties:
+
+- Accuracy
+- Easy to download
+- Easy to integrate
+- Easy to consume
+
+Multiple upstream sources are used to improve accuracy and reduce false negatives. sqlite database containing data in CVE 5.0 schema format is precompiled and distributed as files via ghcr to simplify download. With automatic purl prefix generation even for git repos, searches on the database can be performed with purl, cpe, or even http url string. Every row in the database uses an open specification such as CVE 5.0 or Package URL (purl and vers) thus preventing the possibility of vendor lock-in.
+
+Freeloaders are welcome!
+
 ## Vulnerability Data sources
 
 - Linux [vuln-list](https://github.com/appthreat/vuln-list) (Forked from AquaSecurity)
@@ -109,6 +122,9 @@ vdb --search "npm:gitblame:0.0.1"
 
 # Search by CVE id
 vdb --search CVE-2024-25169
+
+# Search by URL
+vdb --search "https://github.com/electron/electron
 ```
 
 ## License

diff --git a/test/test_utils.py b/test/test_utils.py
@@ -616,14 +616,21 @@ def test_version_len():
 
 
 def test_fix_test():
-    assert utils.fix_text("Unauthenticated crypto and weak IV in Magento\\Framework\\Encryption") == "Unauthenticated crypto and weak IV in Magento Framework Encryption"
+    assert (
+        utils.fix_text(
+            "Unauthenticated crypto and weak IV in Magento\\Framework\\Encryption"
+        )
+        == "Unauthenticated crypto and weak IV in Magento Framework Encryption"
+    )
 
 
 def test_convert_md_references():
     assert utils.convert_md_references(
         "- [Issue #60](https://github.com/tj/node-growl/issues/60)\n- [PR #61](https://github.com/tj/node-growl/pull/61)"
-    ) == [{"name": "Issue #60", "url": "https://github.com/tj/node-growl/issues/60"},
-          {"name": "PR #61", "url": "https://github.com/tj/node-growl/pull/61"}]
+    ) == [
+        {"name": "Issue #60", "url": "https://github.com/tj/node-growl/issues/60"},
+        {"name": "PR #61", "url": "https://github.com/tj/node-growl/pull/61"},
+    ]
 
 
 def test_ubuntu_versions():
@@ -648,7 +655,9 @@ def test_ubuntu_versions():
 
 
 def test_parse_purl():
-    assert utils.parse_purl("pkg:maven/org.jenkins-ci.plugins/JDK_Parameter_Plugin") == {
+    assert utils.parse_purl(
+        "pkg:maven/org.jenkins-ci.plugins/JDK_Parameter_Plugin"
+    ) == {
         "type": "maven",
         "namespace": "org.jenkins-ci.plugins",
         "name": "JDK_Parameter_Plugin",
@@ -692,31 +701,263 @@ def test_parse_purl():
 
 
 def test_purl_vers_convert():
-    assert utils.to_purl_vers("almalinux", [
-        {"version": "0", "status": "affected", "versionType": "semver", "lessThan": "2.3.4"}]) == "vers:rpm/<2.3.4"
-    assert utils.to_purl_vers("almalinux", [{"version": "0", "status": "affected", "versionType": "semver",
-                                             "lessThan": "5.14.21-150500.13.5.1"}]) == "vers:rpm/<5.14.21-150500.13.5.1"
-    assert utils.to_purl_vers("gem", [{'version': '0.1.1', 'status': 'affected', 'versionType': 'gem',
-                                       'lessThanOrEqual': '4.3.5'}]) == "vers:gem/>=0.1.1|<=4.3.5"
+    assert (
+        utils.to_purl_vers(
+            "almalinux",
+            [
+                {
+                    "version": "0",
+                    "status": "affected",
+                    "versionType": "semver",
+                    "lessThan": "2.3.4",
+                }
+            ],
+        )
+        == "vers:rpm/<2.3.4"
+    )
+    assert (
+        utils.to_purl_vers(
+            "almalinux",
+            [
+                {
+                    "version": "0",
+                    "status": "affected",
+                    "versionType": "semver",
+                    "lessThan": "5.14.21-150500.13.5.1",
+                }
+            ],
+        )
+        == "vers:rpm/<5.14.21-150500.13.5.1"
+    )
+    assert (
+        utils.to_purl_vers(
+            "gem",
+            [
+                {
+                    "version": "0.1.1",
+                    "status": "affected",
+                    "versionType": "gem",
+                    "lessThanOrEqual": "4.3.5",
+                }
+            ],
+        )
+        == "vers:gem/>=0.1.1|<=4.3.5"
+    )
     test_tuples = [
-        ("npm", [{'version': '0.0.0', 'status': 'affected', 'versionType': 'npm', 'lessThanOrEqual': '*'}],
-         "vers:npm/*"),
-        ("npm", [{'version': '0.0.0', 'status': 'affected', 'versionType': 'npm', 'lessThan': '1.1.3'}],
-         "vers:npm/>=0.0.0|<1.1.3"),
-        ("npm", [{'version': '0.0.0', 'status': 'affected', 'versionType': 'npm', 'lessThan': '3.5.0'}],
-         "vers:npm/>=0.0.0|<3.5.0"),
-        ("swift", [{'version': '4.0.0-rc.2.5', 'status': 'affected', 'versionType': 'semver', 'lessThan': '4.29.4'}],
-         "vers:swift/>=4.0.0-rc.2.5|<4.29.4"),
-        ("swift", [{'version': '0.0.0', 'status': 'affected', 'versionType': 'semver', 'lessThan': '4.60.3'}],
-         "vers:swift/>=0.0.0|<4.60.3"),
-        ("swift", [{'version': '0.0.0', 'status': 'affected', 'versionType': 'semver', 'lessThan': '4.60.3'}],
-         "vers:swift/>=0.0.0|<4.60.3"),
-        ("maven", [{'version': '8.5.0', 'status': 'affected', 'versionType': 'maven', 'lessThan': '8.5.94'}],
-         "vers:maven/>=8.5.0|<8.5.94"),
-        ("maven", [{'version': '11.0.0-M1', 'status': 'affected', 'versionType': 'maven', 'lessThan': '11.0.0-M12'}],
-         "vers:maven/>=11.0.0-M1|<11.0.0-M12"),
-        ("debian", [{'version': '88.88.8', 'status': 'affected', 'versionType': 'maven'}], "vers:deb/88.88.8"),
-        ("debian", [{'version': '53.5', 'status': 'affected', 'versionType': 'maven', 'lessThanOrEqual': '53.5'}], "vers:deb/53.5"),
+        (
+            "npm",
+            [
+                {
+                    "version": "0.0.0",
+                    "status": "affected",
+                    "versionType": "npm",
+                    "lessThanOrEqual": "*",
+                }
+            ],
+            "vers:npm/*",
+        ),
+        (
+            "npm",
+            [
+                {
+                    "version": "0.0.0",
+                    "status": "affected",
+                    "versionType": "npm",
+                    "lessThan": "1.1.3",
+                }
+            ],
+            "vers:npm/>=0.0.0|<1.1.3",
+        ),
+        (
+            "npm",
+            [
+                {
+                    "version": "0.0.0",
+                    "status": "affected",
+                    "versionType": "npm",
+                    "lessThan": "3.5.0",
+                }
+            ],
+            "vers:npm/>=0.0.0|<3.5.0",
+        ),
+        (
+            "swift",
+            [
+                {
+                    "version": "4.0.0-rc.2.5",
+                    "status": "affected",
+                    "versionType": "semver",
+                    "lessThan": "4.29.4",
+                }
+            ],
+            "vers:swift/>=4.0.0-rc.2.5|<4.29.4",
+        ),
+        (
+            "swift",
+            [
+                {
+                    "version": "0.0.0",
+                    "status": "affected",
+                    "versionType": "semver",
+                    "lessThan": "4.60.3",
+                }
+            ],
+            "vers:swift/>=0.0.0|<4.60.3",
+        ),
+        (
+            "swift",
+            [
+                {
+                    "version": "0.0.0",
+                    "status": "affected",
+                    "versionType": "semver",
+                    "lessThan": "4.60.3",
+                }
+            ],
+            "vers:swift/>=0.0.0|<4.60.3",
+        ),
+        (
+            "maven",
+            [
+                {
+                    "version": "8.5.0",
+                    "status": "affected",
+                    "versionType": "maven",
+                    "lessThan": "8.5.94",
+                }
+            ],
+            "vers:maven/>=8.5.0|<8.5.94",
+        ),
+        (
+            "maven",
+            [
+                {
+                    "version": "11.0.0-M1",
+                    "status": "affected",
+                    "versionType": "maven",
+                    "lessThan": "11.0.0-M12",
+                }
+            ],
+            "vers:maven/>=11.0.0-M1|<11.0.0-M12",
+        ),
+        (
+            "debian",
+            [{"version": "88.88.8", "status": "affected", "versionType": "maven"}],
+            "vers:deb/88.88.8",
+        ),
+        (
+            "debian",
+            [
+                {
+                    "version": "53.5",
+                    "status": "affected",
+                    "versionType": "maven",
+                    "lessThanOrEqual": "53.5",
+                }
+            ],
+            "vers:deb/53.5",
+        ),
     ]
     for tt in test_tuples:
         assert utils.to_purl_vers(tt[0], tt[1]) == tt[2]
+
+
+def test_url_to_purl():
+    assert utils.url_to_purl("https://github.com/owasp-dep-scan/dep-scan") == {
+        "name": "dep-scan",
+        "namespace": "owasp-dep-scan",
+        "qualifiers": None,
+        "subpath": None,
+        "type": "github",
+        "version": None,
+    }
+    assert utils.url_to_purl(
+        "https://github.com/opennms/opennms/commit/8710463077c10034fcfa06556a98fb1a1a64fd0d"
+    ) == {
+        "name": "opennms",
+        "namespace": "opennms",
+        "qualifiers": None,
+        "subpath": None,
+        "type": "github",
+        "version": "8710463077c10034fcfa06556a98fb1a1a64fd0d",
+    }
+    assert utils.url_to_purl(
+        "https://git.eyrie.org/?p=kerberos/remctl.git%3ba=commit%3bh=86c7e44090c988112a37589d2c7a94029eb5e641"
+    ) == {
+        "type": "generic",
+        "namespace": "git.eyrie.org/kerberos",
+        "name": "remctl",
+        "version": "86c7e44090c988112a37589d2c7a94029eb5e641",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://git.savannah.gnu.org/cgit/wget.git/commit/?id=1fc9c95ec144499e69dc8ec76dbe07799d7d82cd"
+    ) == {
+        "type": "generic",
+        "namespace": "git.savannah.gnu.org",
+        "name": "wget",
+        "version": "1fc9c95ec144499e69dc8ec76dbe07799d7d82cd",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://git.gnupg.org/cgi-bin/gitweb.cgi?p=libgcrypt.git%3ba=commit%3bh=9010d1576e278a4274ad3f4aa15776c28f6ba965"
+    ) == {
+        "type": "generic",
+        "namespace": "git.gnupg.org",
+        "name": "libgcrypt",
+        "version": "9010d1576e278a4274ad3f4aa15776c28f6ba965",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://git.openssl.org/gitweb/?p=openssl.git%3ba=commitdiff%3bh=3984ef0b72831da8b3ece4745cac4f8575b19098"
+    ) == {
+        "type": "generic",
+        "namespace": "git.openssl.org",
+        "name": "openssl",
+        "version": "3984ef0b72831da8b3ece4745cac4f8575b19098",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://github.com/rsyslog/librelp/blob/532aa362f0f7a8d037505b0a27a1df452f9bac9e/src/tcp.c#l1205"
+    ) == {
+        "type": "github",
+        "namespace": "rsyslog",
+        "name": "librelp",
+        "version": "532aa362f0f7a8d037505b0a27a1df452f9bac9e",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://github.com/lightsaml/lightsaml/releases/tag/1.3.5"
+    ) == {
+        "type": "github",
+        "namespace": "lightsaml",
+        "name": "lightsaml",
+        "version": "1.3.5",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "https://gitlab.labs.nic.cz/labs/bird/commit/e8bc64e308586b6502090da2775af84cd760ed0d"
+    ) == {
+        "type": "generic",
+        "namespace": "gitlab.labs.nic.cz/labs",
+        "name": "bird",
+        "version": "e8bc64e308586b6502090da2775af84cd760ed0d",
+        "qualifiers": None,
+        "subpath": None,
+    }
+    assert utils.url_to_purl(
+        "http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=0fa3ecd87848c9c93c2c828ef4c3a8ca36ce46c7"
+    ) == {
+        "type": "generic",
+        "namespace": "git.kernel.org/linux",
+        "name": "kernel",
+        "version": "0fa3ecd87848c9c93c2c828ef4c3a8ca36ce46c7",
+        "qualifiers": None,
+        "subpath": None,
+    }
diff --git a/vdb/cli.py b/vdb/cli.py
@@ -96,10 +96,12 @@ def print_results(results):
     table.add_column("Type")
     table.add_column("Namespace")
     table.add_column("Name")
+    table.add_column("Hash")
     table.add_column("Source Data")
     for res in results:
         table.add_row(res.get("cve_id"), res.get("type"),
                       res.get("namespace", ""), res.get("name"),
+                      res.get("source_data_hash"),
                       Syntax(orjson.dumps(
                           res.get("source_data").model_dump(mode="json", exclude_none=True),
                           option=orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE).decode("utf-8", errors="ignore"), "json", word_wrap=True))
@@ -138,6 +140,8 @@ def main():
             results = search.search_by_purl_like(args.search, with_data=True)
         elif args.search.startswith("CVE-") or args.search.startswith("GHSA-") or args.search.startswith("MAL-"):
             results = search.search_by_cve(args.search, with_data=True)
+        elif args.search.startswith("http"):
+            results = search.search_by_url(args.search, with_data=True)
         else:
             results = search.search_by_cpe_like(args.search, with_data=True)
         if results: