feat: add malshare website as a new backend

your_daily_dose_malware/app/utils/http_utils.py

@@ -23,8 +23,7 @@ class HttpUtils:
     @staticmethod
     def get(url, headers=None, params=None, timeout=10):
         return HttpUtils._make_request(
-            HttpMethod.GET, url, headers=headers, params=params,
-            timeout=timeout
+            HttpMethod.GET, url, headers=headers, params=params, timeout=timeout
         )
 
     @staticmethod
@@ -52,8 +51,7 @@ def put(url, headers=None, params=None, data=None, json=None, timeout=10):
         )
 
     @staticmethod
-    def delete(url, headers=None, params=None, data=None, json=None,
-               timeout=10):
+    def delete(url, headers=None, params=None, data=None, json=None, timeout=10):
         return HttpUtils._make_request(
             HttpMethod.DELETE,
             url,
@@ -66,8 +64,8 @@ def delete(url, headers=None, params=None, data=None, json=None,
 
     @staticmethod
     def _make_request(
-        method, url, headers=None, params=None, data=None, json=None,
-            timeout=10):
+        method, url, headers=None, params=None, data=None, json=None, timeout=10
+    ):
         try:
             response = requests.request(
                 method=method,

your_daily_dose_malware/backends/malshare.py

@@ -0,0 +1,46 @@
+import os
+
+import requests
+import click
+from dotenv import load_dotenv
+from rich.progress import Progress
+
+load_dotenv()
+
+
+def scrap_malshare():
+    with Progress() as progress:
+        api_key = os.getenv("MALSHARE_API_KEY")
+        print(api_key)
+        try:
+            response = requests.post(
+                "https://malshare.com/api.php",
+                verify=True,
+                params={"api_key": api_key, "action": "getlist"},
+            )
+            response.raise_for_status()
+            hashes = response.json()
+        except Exception as err:
+            raise err
+        try:
+            print(f"  - Malshare: {len(hashes)} Samples")
+            sha256_ids = [hashe["sha256"] for hashe in hashes]
+            task = progress.add_task(
+                "  - [green]Downloading Malware Files...", total=len(sha256_ids)
+            )
+            click.echo(f"	Your query yield {len(sha256_ids)} results")
+            for sha256_id in sha256_ids:
+                response = requests.post(
+                    "https://malshare.com/api.php",
+                    params={"api_key": api_key, "action": "getfile", "hash": sha256_id},
+                    verify=True,
+                )
+                response.raise_for_status()
+                with open(f"malware_{sha256_id[:4]}.zip", "wb") as f:
+                    f.write(response.content)
+                    progress.update(task, advance=1)
+        except Exception as err:
+            raise err
+
+
+scrap_malshare()

your_daily_dose_malware/backends/malware_bazaar.py

@@ -0,0 +1,92 @@
+import click
+import requests
+import os
+from rich.progress import Progress
+
+
+def hundred_most_recent(headers_info):
+    # TODO Pending Refactor this method to match all_most_recent style
+    response = requests.post(
+        "https://mb-api.abuse.ch/api/v1/",
+        data={"query": "get_recent", "selector": "100"},
+        headers=headers_info,
+    )
+    json_response = response.json()
+
+    if json_response["query_status"] == "no_selector":
+        click.echo(
+            "No selector provided. Please use either time or limit " "as selector",
+            color=True,
+        )
+    if json_response["query_status"] == "unknown_selector":
+        click.echo("Selector unknown. Please use either time or limit as selector")
+    if json_response["query_status"] == "no_results":
+        click.echo("Your query yield no results")
+    else:
+        data_length = len(json_response["data"])
+        click.echo(f"	Your query yield {data_length} results")
+        for data in json_response["data"]:
+            sha256_name = data["sha256_hash"]
+            if os.path.exists(f"malware_{sha256_name[:4]}.zip"):
+                continue
+            response = requests.post(
+                "https://mb-api.abuse.ch/api/v1/",
+                data={"query": "get_file", "sha256_hash": sha256_name},
+                headers=headers_info,
+            )
+            with open(f"malware_{sha256_name[:4]}.zip", "wb+") as f:
+                f.write(response.content)
+                click.echo(f"	malware_{sha256_name[:4]}.zip downloaded")
+
+
+def all_most_recent(headers_info):
+
+    with Progress() as progress:
+
+        response = requests.post(
+            "https://bazaar.abuse.ch/export/txt/sha256/recent", headers=headers_info
+        )
+
+        with open("sha256_names.txt", "wb+") as f:
+            f.write(response.content)
+            f.seek(0)
+            new_hashes = list()
+            file_lines = [line.strip() for line in f.readlines()]
+            for index, line in enumerate(file_lines, start=0):
+                # skip the first 9 lines and last line
+                if 8 < index < len(file_lines) - 1:
+                    new_hashes.append(line)
+                continue
+
+            print(f"  - Malware Bazaar: {len(new_hashes)} Samples")
+
+        with open("sha256_names.txt", "w") as f:
+            for line in new_hashes:
+                f.write(line.decode("utf-8") + "\n")
+        sha256_names = open("sha256_names.txt", "r").readlines()
+
+        task = progress.add_task(
+            "  - [green]Downloading Malware Files...", total=len(sha256_names)
+        )
+
+        for sha256_hash in sha256_names:
+
+            if os.path.exists(f"malware_{sha256_hash[:4]}.zip"):
+                continue
+            response = requests.post(
+                "https://mb-api.abuse.ch/api/v1/",
+                data={"query": "get_file", "sha256_hash": sha256_hash},
+                headers=headers_info,
+            )
+            with open(f"malware_{sha256_hash[:4]}.zip", "wb") as f:
+                f.write(response.content)
+
+            progress.update(task, advance=1)
+
+
+def scrap_malware_bazaar(hundred_recent):
+    headers = {"API-KEY": os.getenv("MALWARE_BAZAAR_API_KEY")}
+    if hundred_recent:
+        hundred_most_recent(headers)
+    else:
+        all_most_recent(headers)

your_daily_dose_malware/commands/utils/retrieve_malware.py

@@ -15,8 +15,7 @@ def hundred_most_recent(headers_info):
 
     if json_response["query_status"] == "no_selector":
         click.echo(
-            "No selector provided. Please use either time or limit "
-            "as selector",
+            "No selector provided. Please use either time or limit " "as selector",
             color=True,
         )
     if json_response["query_status"] == "unknown_selector":
@@ -45,8 +44,7 @@ def all_most_recent(headers_info):
     with Progress() as progress:
 
         response = requests.post(
-            "https://bazaar.abuse.ch/export/txt/sha256/recent",
-            headers=headers_info
+            "https://bazaar.abuse.ch/export/txt/sha256/recent", headers=headers_info
         )
 
         with open("sha256_names.txt", "wb+") as f:
@@ -67,8 +65,9 @@ def all_most_recent(headers_info):
                 f.write(line.decode("utf-8") + "\n")
         sha256_names = open("sha256_names.txt", "r").readlines()
 
-        task = progress.add_task("  - [green]Downloading Malware Files...",
-                                 total=len(sha256_names))
+        task = progress.add_task(
+            "  - [green]Downloading Malware Files...", total=len(sha256_names)
+        )
 
         for sha256_hash in sha256_names:
 

your_daily_dose_malware/main.py

@@ -1,48 +1,46 @@
-import os
 import argparse
-from rich.console import Console
-from your_daily_dose_malware.commands.utils.retrieve_malware import (all_most_recent,
-                                                                     hundred_most_recent)
-from your_daily_dose_malware.constants import (MALWARE_BANNER,
-                                               MALWARE_DESCRIPTION)
 
 from dotenv import load_dotenv
+from rich.console import Console
 
+from your_daily_dose_malware.backends import malware_bazaar, malshare
+from your_daily_dose_malware.constants import MALWARE_BANNER, MALWARE_DESCRIPTION
 
 load_dotenv()
 
 
-def scrap_malware_bazaar(hundred_recent):
-    headers = {"API-KEY": os.getenv("MwBz_API_KEY")}
-    if hundred_recent:
-        hundred_most_recent(headers)
-    else:
-        all_most_recent(headers)
-
-
 def run_scrapper(args):
-    scrap_malware_bazaar(args.hundred_recent)
+    malware_bazaar.scrap_malware_bazaar(args.hundred_recent)
+    malshare.scrap_malshare()
 
 
 def main():
-
     parser = argparse.ArgumentParser(
-        description="Download recent samples from multiple OSINT provider backends"
-                    "MalwareBazaar: all the recent recent malware's uploaded within"
-                    "the last 48 hours sha256 hash"
-                    "MalwareBazaar: By using api from 'https://mb-api.abuse.ch/api/v1/'"
+        description="""Download recent samples from multiple OSINT provider
+         backends
+        "MalwareBazaar: all the recent recent malware's uploaded within"
+        "the last 48 hours sha256 hash"
+        "MalwareBazaar: By using api from 'https://mb-api.abuse.ch/api/v1/'"""
+    )
+    parser.add_argument(
+        "-MwBz_48H",
+        "--last-48H",
+        action="store_true",
+        help="get the most recent sha256 hashes",
+        default=True,
+    )
+    parser.add_argument(
+        "-MwBz_100",
+        "--hundred-recent",
+        action="store_true",
+        help="""get the most recent hundred recent malware's
+         uploaded within the last 60 min""",
     )
-    parser.add_argument('-MwBz_48H', '--last-48H', action='store_true',
-                        help="get the most recent sha256 hashes", default=True)
-    parser.add_argument('-MwBz_100', '--hundred-recent',
-                        action='store_true',
-                        help="get the most recent hundred recent malware's uploaded within the last 60 min")
     args = parser.parse_args()
 
     console = Console()
     console.print(MALWARE_BANNER)
     console.print(f"{MALWARE_DESCRIPTION} \n")
-
     run_scrapper(args)