diff --git a/config.example b/config.example new file mode 100644 index 0000000..f198ba0 --- /dev/null +++ b/config.example @@ -0,0 +1,4 @@ +shodan_api: '' +shodan_query: +data_retention: 90 #days +scan_interval: 30 #days diff --git a/library/__pycache__/jsonDataStore_lib.cpython-312.pyc b/library/__pycache__/jsonDataStore_lib.cpython-312.pyc new file mode 100644 index 0000000..7bacdf9 Binary files /dev/null and b/library/__pycache__/jsonDataStore_lib.cpython-312.pyc differ diff --git a/library/__pycache__/shodan_lib.cpython-312.pyc b/library/__pycache__/shodan_lib.cpython-312.pyc new file mode 100644 index 0000000..3be4105 Binary files /dev/null and b/library/__pycache__/shodan_lib.cpython-312.pyc differ diff --git a/library/jsonDataStore_lib.py b/library/jsonDataStore_lib.py new file mode 100644 index 0000000..ea4dcb5 --- /dev/null +++ b/library/jsonDataStore_lib.py @@ -0,0 +1,141 @@ +import os +import json +from datetime import datetime, timedelta, timezone +from colorama import Fore, Back, Style, init + +############################################################################## +# jsonDataStore Class +# Description: VERY basic way of storing information in a json formatted text file. +# Long term probably need to convert this to something else.. sql-lite? i dunno. +############################################################################## +class jsonDataStore: + dataStore={} + datestoreFilename='' + logger='' + + + # Creation Method: Nothing really going on here other than creating object + def __init__(self, filename, logger): + self.logger=logger + init(autoreset=True) + print (Fore.GREEN + f' [*]: Data Store Object Created') + self.logger.debug(" [*]: Data Store Object Created") + self.datestoreFilename=filename + + #self.getDataStore(filename) + + # getDataStore Method: returns data stored in class/method + def getDataStore(self): + return self.dataStore.copy() + + # readDataStoreFromFile: reads json datastore master file + def readDataStoreFromFile(self, file_path): + file_path=self.datestoreFilename + self.check_file_exists(file_path) + + try: + # Open and read the JSON file + with open(file_path, 'r') as file: + data = json.load(file) + self.logger.debug(" [+]: reading data from file.") + self.dataStore=data.copy() + return data + except FileNotFoundError: + print(Fore.RED + f" [+]: The file {file_path} does not exist.") + self.logger.debug("[+]: The file does not exist.") + except json.JSONDecodeError as e: + print(Fore.RED + f" [-]: Invalid JSON in file: {file_path}. Error: {e}") + self.logger.debug(f"[-]: Invalid JSON in file: {file_path}. Error: {e}") + except Exception as e: + print(Fore.RED + f"An error occurred: {e}") + self.logger.debug(f"[-]: An error occurred: {e}") + return None + + # addDataToStore: adds new shodan data to datestore. + def addDataToStore(self, data_key, data_to_store): + if self.dataStore.get(data_key): #already in DB + dataFromDictionary=self.dataStore.get(data_key) + + # converts text timestamp to datetime stamp so you can compare + firstseen_timestamp = self.convertStrTimeStamptoDateTime(self.dataStore[data_key]['first_seen']) + lastseen_timestamp = self.convertStrTimeStamptoDateTime(self.dataStore[data_key]['last_seen']) + + # converts text timestamp from new entry to datetime stamp so you can compare + data_to_store_timestamp = self.convertStrTimeStamptoDateTime(data_to_store['timestamp']) + + # Because python reads files in a random order not alphabetically or by date, you gotta compare dates with + # each item read. (only applies to reading old data files) + if firstseen_timestamp > data_to_store_timestamp: + dataFromDictionary['first_seen']=data_to_store['timestamp'] + + if lastseen_timestamp < data_to_store_timestamp: + dataFromDictionary['last_seen']=data_to_store['timestamp'] + + dataFromDictionary['seen_count']+=1 + self.dataStore[data_key]=dataFromDictionary.copy() + self.logger.info(f" [+]: Updated Entry: {data_key}") + + else: # new entry + self.dataStore[data_key] = {} + + data_to_store['first_seen'] = data_to_store['timestamp'] + data_to_store['last_seen'] = data_to_store['timestamp'] + data_to_store['last_scan'] = 0 + data_to_store['seen_count'] = 1 + data_to_store['vulnerability_count']: len(data_to_store['vulns']) + + self.dataStore[data_key] = data_to_store.copy() + self.logger.info(f" [+]: New Entry: {data_key}") + data = {} + + # deleteFromDataStore: delete entry in data store by key. used for pruning + # old entries in the database + def deleteFromDataStore(self, key): + print (f'deleting from store {key}') + self.logger.info(f' [+]: Deleting from store {key}') + + # countRecords: returns number of records in data store + def countRecords(self): + return len(self.dataStore) + + # saveDataStore: saves all data in data store to json file + def saveDataStore(self, filename): + data=self.dataStore + file_path=self.datestoreFilename + try: + # Open and write to the JSON file + with open(file_path, 'w') as file: + json.dump(self.dataStore, file, indent=4) + print(Fore.GREEN + f"[*]: Data successfully written to {file_path}.") + print (Fore.GREEN + f' [+]: Number of Records saved: {self.countRecords()}') + self.logger.debug(f" [*]: Data successfully written to {file_path}.") + self.logger.debug(f' [+]: Number of Records saved: {self.countRecords()}') + + except Exception as e: + print(f"An error occurred: {e}") + self.logger.info(f"[-]: An error occurred: {e}") + + # check_file_exists: just checks if the data store file exists, if not creates one + def check_file_exists(self, file_path): + # Check if the file exists + if not os.path.exists(file_path): + # Create the file + with open(file_path, 'w') as file: + # Optionally write initial content to the file + file.write('') + print(Fore.GREEN + f" [+]: DataStore {file_path} created.") + self.logger.debug(f" [+]: DataStore {file_path} created.") + else: + print(Fore.YELLOW + f" [+]: DataStore {file_path} already exists.") + self.logger.debug(f" [+]: DataStore {file_path} already exists.") + + # convertStrTimeStamptoDateTime: datastore file saves date as text, this converts + # back to python datetime + def convertStrTimeStamptoDateTime(self, strTimeStamp): + datetime_obj = datetime.fromisoformat(strTimeStamp) + + # Format the datetime object to the desired format + # For example, converting to 'YYYY-MM-DD HH:MM:SS' format + formatted_timestamp = datetime.strptime(strTimeStamp, '%Y-%m-%dT%H:%M:%S.%f') + + return formatted_timestamp diff --git a/library/shodan_lib.py b/library/shodan_lib.py new file mode 100644 index 0000000..713ee65 --- /dev/null +++ b/library/shodan_lib.py @@ -0,0 +1,59 @@ +from datetime import datetime, timedelta, timezone +import shodan +from colorama import Fore, Back, Style, init + +''' +shodan_api_class: just a wrapper for shodan api + +__init__ : takes in the shodan api and verifies communication +check_api : verifies communication with shodan +query_shodan : queries shodan with given query +''' +class shodan_api_class: + shodan_api_key='' + shodan_query='' + shodan_valid_key=False + shodan_obj='' + logger='' + def __init__(self, shodan_api_key, logger): + self.logger=logger + init(autoreset=True) + print (Fore.GREEN + "Initializing Shodan") + self.shodan_api_key=shodan_api_key + self.shodan_obj=shodan.Shodan(self.shodan_api_key) + + self.shodan_valid_key=self.check_api(self) + print (Fore.YELLOW + f' [+]: Shodan Communication is: {self.shodan_valid_key}') + + def check_api(self,shodan_obj): + try: + results = self.shodan_obj.info() + + if results: + self.logger.debug(f" [+]: Shodan API is valid") + return True + else: + self.logger.debug(f" [-]: Shodan API is NOT valid") + return False + except shodan.APIError as e: + print(Fore.RED + f"Error: {e}") + self.logger.ERROR(f" [+]: Shodan API Error: {e}") + return False + + def query_shodan(self, shodan_query): + print (Fore.CYAN + f' [+]: Querying: {shodan_query}') + self.logger.info(f' [+]: Querying Shodan: {shodan_query}') + + # Define the query parameters + + # Perform the search query + try: + results = self.shodan_obj.search(shodan_query) + # Print the results + print(Fore.GREEN + f" [+]: Results found: {results['total']}") + self.logger.info(f' [+]: Results found: {results['total']}') + + return results.copy() + except shodan.APIError as e: + print(Fore.RED + f"Error: {e}") + self.logger.ERROR(f"Error: {e}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3f2764c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +art==6.2 +certifi==2024.6.2 +charset-normalizer==3.3.2 +click==8.1.7 +click-plugins==1.1.1 +colorama==0.4.6 +filelock==3.15.1 +fire==0.6.0 +idna==3.7 +pyfiglet==1.0.2 +PyYAML==6.0.1 +requests==2.32.3 +requests-file==2.1.0 +shodan==1.31.0 +six==1.16.0 +tabulate==0.9.0 +termcolor==2.4.0 +text2art==0.2.0 +tldextract==5.1.2 +urllib3==2.2.1 +XlsxWriter==3.2.0 diff --git a/shodanDataStore.json b/shodanDataStore.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/shodanDataStore.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/shodanPull.py b/shodanPull.py new file mode 100644 index 0000000..d26679c --- /dev/null +++ b/shodanPull.py @@ -0,0 +1,276 @@ +#this is rough.. but works +import argparse +from datetime import datetime, timedelta, timezone +import datetime +import json +import yaml +import os +import shutil +from library import shodan_lib +from library import jsonDataStore_lib +from tabulate import tabulate +from colorama import Fore, Back, Style, init +import logging +import argparse +from art import text2art + +from library.shodan_lib import shodan_api_class +# --===============================================================-- +# Gather +# --===============================================================-- +def gather(config, dataStore, logger): + shodan_obj=shodan_lib.shodan_api_class(shodan_api_key=config['shodan_api'], logger=logger) + + # Get the current UTC date and time + now_utc = datetime.datetime.now(timezone.utc) + + # Calculate yesterday's date in UTC + yesterday_utc = now_utc - timedelta(days=3) + + # Format yesterday's UTC date as a string + yesterday_utc_str = yesterday_utc.strftime("%Y-%m-%d") + now_utc_str = now_utc.strftime("%Y-%m-%d") + + print (yesterday_utc_str) + + # Define the search query with the timestamp and geographical filters + query = f'after:{yesterday_utc_str} country:US state:LA' + current_date = datetime.datetime.now() + + results=shodan_obj.query_shodan(query) + ProcessShodanResults(results['matches'], dataStore) + +# --===============================================================-- +# Hunt +# --===============================================================-- +def hunt(config, dataStore): + print (Fore.GREEN + f'[*]: Hunting through Shodan Data') + print (Fore.GREEN + f' [+]: Hunter has {dataStore.countRecords()} of records to go through') + + data=dataStore.getDataStore() + + #Pruning Data Store (removing old/dead records) + print(Fore.GREEN + f' [+]: Pruning records that are over {config['data_retention']} days old.') + for record in data: + firstLastDelta = (dataStore.convertStrTimeStamptoDateTime(data[record]['last_seen']) - dataStore.convertStrTimeStamptoDateTime(data[record]['first_seen'])) + + if firstLastDelta.days > config['data_retention']: + #TODO: pop dead records + print (Fore.RED + " Kill It [-]: record ",":", firstLastDelta.days) + + if (data[record]['last_scan'] == 0) or (datetime.now() - dataStore.convertStrTimeStamptoDateTime(data[record]['last_scan']) > config['scan_interval']): + #TODO: scan here + #TODO: put current date in last_scan + print (f'Scan here') + +# --===============================================================-- +# Show +# --===============================================================-- +def show(config, dataStore): + print(Fore.GREEN + f'[*]: Displaying Shodan Data') + print(Fore.GREEN + f' [+]: Displaying {dataStore.countRecords()} records') + + data = dataStore.getDataStore() + + headers = ['IP', 'First Seen Date', 'Last Seen Date', 'Last Seen -> Today', 'Last Scan Date', 'Vulnerability Count','Seen Count','Location'] + tableData = [] + table_row_count=0 + + date_str=str(datetime.datetime.now()) + file_str="table_"+date_str+".txt" + file_str=file_str.replace(" ","_") + + for record in data: + + dateDiff = datetime.datetime.now() - dataStore.convertStrTimeStamptoDateTime(data[record]['last_seen']) + vulnCount = len(data[record]['vuln_list']) + + if dateDiff.days < 15: + location_str=data[record]['location']['city']+","+data[record]['location']['region_code'] + + row = [data[record]['ip_str'], data[record]['first_seen'], + data[record]['last_seen'], dateDiff, data[record]['last_scan'], + vulnCount, data[record]['seen_count'],location_str] + tableData.append(row) + table_row_count+=1 + + print(tabulate(tableData, headers, tablefmt="pretty")) + + if table_row_count>50: + print (f' [+]: large amount of records, saving output to table.txt') + print (f' [+]: records saved: {table_row_count}') + finalTable=tabulate(tableData, headers, tablefmt="pretty") + with open(file_str, "w") as file: + file.write(finalTable) + + +# --===============================================================-- +# Process Shodan Results +# --===============================================================-- +def ProcessShodanResults(results, dataStore): + dictEntry={} + + #pulling out important fields and throwing out the crap + for result in results: + if result.get('vulns'): #if there is a vulnerability add it to list + if result.get('ip_str'): + dictEntry['timestamp'] = result.get('timestamp') + dictEntry['ip_str']=result.get('ip_str') + dictEntry['port'] = result.get('port') + dictEntry['version'] = result.get('version') + dictEntry['location'] = result.get('location') + dictEntry['ip'] = result.get('ip') + dictEntry['product'] = result.get('product') + dictEntry['timestamp'] = result.get('timestamp') + dictEntry['hostnames'] = result.get('hostnames') + dictEntry['org'] = result.get('org') + dictEntry['isp'] = result.get('isp') + dictEntry['os'] = result.get('os') + dictEntry['vuln_list'] = list(set(result['vulns'].keys())) + + print (Fore.GREEN + f' [+ Had vulnerability +]: : {result['ip_str']} to dataStore') + logger.info(f' [+ Had vulnerability +]: : {result['ip_str']} added to dataStore') + dataStore.addDataToStore(data_key=dictEntry['ip_str'], data_to_store=dictEntry.copy()) + else: + print(Fore.YELLOW + f' [- No Vulnerability -]: : {result['ip_str']} not added') + logger.info(f' [- No Vulnerability -]: : {result['ip_str']} not added to data store') + +# --===============================================================-- +# Read JSON Files from folder +# --===============================================================-- +def list_json_files(folder_path): + # Get a list of all files in the specified folder + files = os.listdir(folder_path) + + # Filter the list to include only JSON files + json_files = [f for f in files if f.endswith('.json')] + + return json_files + +def read_json_file(file_path): + json_objects = [] + try: + # Open the text file + with open(file_path, 'r') as file: + # Read each line in the file + for line in file: + # Strip any extra whitespace and parse the JSON object + json_object = json.loads(line.strip()) + json_objects.append(json_object) + + except FileNotFoundError: + print(Fore.GREEN + f"The file {file_path} does not exist.") + except json.JSONDecodeError as e: + print(Fore.GREEN + f"Invalid JSON on line: {line.strip()}. Error: {e}") + except Exception as e: + print(Fore.GREEN + f"An error occurred: {e}") + + return json_objects + +def processShodanJSONFiles(folderToProcess): + fileWithPath=[] + + print (Fore.GREEN + f' [+]: folder being processed: {folderToProcess}') + jsonFileList = list_json_files(folderToProcess) + + # adding folder to filename for processing + folderToProcess=folderToProcess+'/' + for item in jsonFileList: + item=folderToProcess+item + fileWithPath.append(item) + + for fileItem in fileWithPath: + print (Fore.GREEN + f' [+]: Processing: {fileItem}') + jsonFile=read_json_file(fileItem) + + ProcessShodanResults(jsonFile, dataStore) + +# --===============================================================-- +# Load Config +# --===============================================================-- +def load_config(file_path): + #todo: check for file, if no file, create it + + with open(file_path, 'r') as file: + config = yaml.safe_load(file) + return config + +def check_config_exists(): + source_file = 'config.example' + target_file = 'config.yml' + + if not os.path.exists(target_file): + if os.path.exists(source_file): + shutil.copyfile(source_file, target_file) + print(Fore.GREEN + f"Copied {source_file} to {target_file}.") + print (Fore.GREEN + f"Error: {target_file} Configuration file did not exist.") + print (Fore.GREEN + f"{target_file} was created, please edit with text editor for your configuration") + return False + else: + print(Fore.GREEN + f"Source file {source_file} does not exist.") + return False + else: + print(Fore.YELLOW + f" [+]: {target_file} already exists.") + return True + +# --===============================================================-- +# Main +# --===============================================================-- + +if __name__ == "__main__": + init() + + # Configure logging settings + logging.basicConfig( + level=logging.INFO, # Set the logging level to DEBUG + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', # Log message format + filename='shodanPull_v2.log', # Log file name + filemode='a' # Append mode + ) + + logger = logging.getLogger('shodanPull_v2 Logger') + logger.info(f'--===========================================--') + + config_exists=check_config_exists() + + if config_exists: + print(Fore.BLUE + f"--===============================================================--") + print(Fore.BLUE + f"--== qShodan ==--") + print(Fore.BLUE + f"--===============================================================--") + + dataStore = jsonDataStore_lib.jsonDataStore('./shodanDataStore.json', logger) + dataStore.readDataStoreFromFile('./shodanDataStore.json') + + config = load_config('config.yml') + + gather(config, dataStore, logger) + + #show(config, dataStore) + + # parser = argparse.ArgumentParser(description='A tool with gather, hunt, and show functionalities.') + # + # subparsers = parser.add_subparsers(dest='command', help='Sub-command help') + # + # # Gather command + # parser_gather = subparsers.add_parser('gather', help='Gather data') + # parser_gather.set_defaults(func=gather) + # + # # Hunt command + # parser_hunt = subparsers.add_parser('hunt', help='Hunt targets') + # parser_hunt.set_defaults(func=hunt) + # + # # Show command + # parser_show = subparsers.add_parser('show', help='Show results') + # parser_show.set_defaults(func=show) + # + # # Parse the arguments + # args = parser.parse_args() + + # Execute the appropriate function based on the command + # if hasattr(args, 'func'): + # args.func() + # else: + # parser.print_help() + + dataStore.saveDataStore('./shodanDataStore.json') + logger.info(f'--===========================================--') diff --git a/shodanPull_v2.log b/shodanPull_v2.log new file mode 100644 index 0000000..ae3246a --- /dev/null +++ b/shodanPull_v2.log @@ -0,0 +1,11 @@ +2024-07-23 14:52:44,978 - shodanPull_v2 Logger - INFO - --===========================================-- +2024-07-23 14:53:41,688 - shodanPull_v2 Logger - INFO - --===========================================-- +2024-07-23 14:53:41,957 - shodanPull_v2 Logger - INFO - [+]: Querying Shodan: after:2024-07-20 country:US state:LA +2024-07-23 14:53:46,669 - shodanPull_v2 Logger - INFO - [+]: Results found: 1 +2024-07-23 14:53:46,671 - shodanPull_v2 Logger - INFO - [- No Vulnerability -]: : 199.19.233.115 not added to data store +2024-07-23 14:53:46,672 - shodanPull_v2 Logger - INFO - --===========================================-- +2024-07-23 14:54:41,166 - shodanPull_v2 Logger - INFO - --===========================================-- +2024-07-23 14:54:42,010 - shodanPull_v2 Logger - INFO - [+]: Querying Shodan: after:2024-07-20 country:US state:LA +2024-07-23 14:54:43,146 - shodanPull_v2 Logger - INFO - [+]: Results found: 1 +2024-07-23 14:54:43,147 - shodanPull_v2 Logger - INFO - [- No Vulnerability -]: : 199.19.233.115 not added to data store +2024-07-23 14:54:43,148 - shodanPull_v2 Logger - INFO - --===========================================--