Merge pull request DedSecInside#44 from KingAkeem/dev

PEP8 code and fixed some minor bugs

.gitignore

@@ -1,2 +1,8 @@
 modules/__pycache__/
 .*.swp
+.ropeproject/
+modules/.ropeproject/
+tests/__pycache__/
+modules/__init__.py
+.idea/
+tests/.ropeproject/

modules/bcolors.py

@@ -1,16 +1,14 @@
-__all__ = ['Bcolors']
-
-
 class Bcolors:
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-    WHITE = '\033[97m'
-    On_Black = '\033[40m'
-    On_Red = '\033[41m'
-
+
+    def __init__(self):
+        self.HEADER = '\033[95m'
+        self.OKBLUE = '\033[94m'
+        self.OKGREEN = '\033[92m'
+        self.WARNING = '\033[93m'
+        self.FAIL = '\033[91m'
+        self.ENDC = '\033[0m'
+        self.BOLD = '\033[1m'
+        self.UNDERLINE = '\033[4m'
+        self.WHITE = '\033[97m'
+        self.On_Black = '\033[40m'
+        self.On_Red = '\033[41m'

modules/getemails.py

@@ -1,35 +1,36 @@
-import sys
-import os
-sys.path.append(os.path.abspath('../'))
 from modules.bcolors import Bcolors
+from bs4 import BeautifulSoup
 from modules.savefile import saveJson
-import bs4
 
-__all__ = ['getMails']
 
 """Get all emails from the website"""
 
-def getMails(soup,save=0):
-    _soup_instance = bs4.BeautifulSoup
+
+def getMails(soup, save=0):
+    b_colors = Bcolors()
+    _soup_instance = BeautifulSoup
     if isinstance(type(soup), type(_soup_instance)):
         emails = []
         for link in soup.find_all('a'):
             email_link = link.get('href')
-            if email_link != None:
+            if email_link is not None:
                 if 'mailto' in email_link:
                     """Split email address on"""
                     email_addr = email_link.split(':')
                     emails.append(email_addr[1])
             else:
                 pass
         """Pretty print output as below"""
-        print ('') 
-        print (Bcolors.OKGREEN+'Mails Found - '+Bcolors.ENDC+str(len(emails)))
+        print ('')
+        print (b_colors.OKGREEN+'Mails Found - '+b_colors.ENDC+str(len(emails)))
         print ('-------------------------------')
         for mail in emails:
             print (mail)
         if save:
-            saveJson("Extracted-Mail-IDs",emails)
+            saveJson("Extracted-Mail-IDs", emails)
         return ''
     else:
-        raise(Bcolors.FAIL+'Method parameter is not of instance bs4.BeautifulSoup'+Bcolors.ENDC)
+        msg = ''.join((b_colors.FAIL,
+                       'Method parameter is not of instance BeautifulSoup',
+                       b_colors.ENDC))
+        raise(msg)

modules/getweblinks.py

@@ -1,83 +1,47 @@
-import sys
-import os
-sys.path.append(os.path.abspath('../'))
-from modules.savefile import saveJson 
-import urllib.request 
+import urllib.request
 from modules.bcolors import Bcolors
-import bs4
-import time
-import threading
-import http
+from bs4 import BeautifulSoup
 
-__all__ = ['getLinks']
 
 
-def link_status(web,out_queue,index):
-    link_live = False
-    live_links = 0
+def link_status(web, out_queue, index):
+    b_colors = Bcolors()
     out_queue[index] = web + " is_live = False "
     try:
-        urllib.request.urlopen(web)    
-        link_live = True
+        urllib.request.urlopen(web)
         out_queue[index] = web + " is_live = True "
-        live_links += 1
         print(web)
-    except urllib.error.HTTPError as e:
-        print(Bcolors.On_Red+web+Bcolors.ENDC)
-    except urllib.error.URLError as e:
-        print(Bcolors.On_Red+web+Bcolors.ENDC)
-    except http.client.RemoteDisconnected as e:
-        print(Bcolors.On_Red+web+Bcolors.ENDC)
-    return
+    except urllib.error.HTTPError:
+        print(b_colors.On_Red+web+b_colors.ENDC)
 
 
-"""Get all onion links from the website"""
-def getLinks(soup,ext,live=0,save=0):
-    _soup_instance = bs4.BeautifulSoup
+def getLinks(soup, ext, live=0, save=0):
+
+    """Get all onion links from the website"""
+
+    b_colors = Bcolors()
     extensions = []
     if ext:
         for e in ext:
-            extensions.append(e) 
-    if isinstance(type(soup), type(_soup_instance)):
+            extensions.append(e)
+
+    if isinstance(type(soup), type(BeautifulSoup)):
         websites = []
-        start_time = time.time()
+
         for link in soup.find_all('a'):
             web_link = link.get('href')
-            if web_link != None:
-                if ('http' in web_link or 'https' in web_link):
-                    if ext:
-                        for exten in extensions:
-                            if web_link.endswith(exten):
-                                websites.append(web_link)
-                    else:
-                        websites.append(web_link)            
-            else:
-                pass
+            if web_link and ('http' in web_link or 'https' in web_link):
+
+                for exten in extensions:
+                    if web_link.endswith(exten):
+                        websites.append(web_link)
+                else:
+                    websites.append(web_link)
         """Pretty print output as below"""
-        print ('') 
-        print (Bcolors.OKGREEN+'Websites Found - '+Bcolors.ENDC+str(len(websites)))
-        print ('-------------------------------')
-        if live:
-            threads = []
-            result = [{} for x in websites]
-            for web in websites:
-                t = threading.Thread(target=link_status, args=(web,result,websites.index(web)))
-                t.start()
-                threads.append(t)
-            try:
-                for t in threads:
-                    t.join()
-                if save:
-                    saveJson("Live-Onion-Links",result)
-            except:
-                pass
-        else:
-            for web in websites:
-                print(web)
-            if save:
-                saveJson("Onion-Links",websites)
+        print(''.join((b_colors.OKGREEN,
+              'Websites Found - ', b_colors.ENDC, str(len(websites)))))
+        print('-------------------------------')
         return websites
-        #print (Bcolors.OKGREEN+'Live Links:'+live_links+'  Dead Links:'+(str(len(websites))-live_links))
-        #print ('-------------------------------')           
+
     else:
-        raise('Method parameter is not of instance bs4.BeautifulSoup')
+        raise('Method parameter is not of instance BeautifulSoup')

modules/info.py

@@ -1,87 +1,73 @@
-import urllib.request
-from urllib.parse import urlsplit
-from termcolor import colored, cprint
-
-__all__ = ['executeAll']
-
-
-def __init__(self):
-    pass
-
+import requests
 
+from urllib.parse import urlsplit
+from termcolor import cprint
 
 
 def executeAll(target):
     try:
-        self.get_robots_txt(target)
-    except:
-        cprint("No robots.txt file Found!", "blue")
+        get_robots_txt(target)
+    except Exception:
+        cprint("No robots.txt file Found!""blue")
     try:
-        self.get_dot_git(target)
-    except:
-        cprint("Error !", "red")
+        get_dot_git(target)
+    except Exception:
+        cprint("Error !""red")
     try:
-        self.get_dot_svn(target)
-    except:
-        cprint("Error", "red")
+        get_dot_svn(target)
+    except Exception:
+        cprint("Error""red")
     try:
-        self.get_dot_htaccess(target)
-    except:
-        cprint("Error", "red")
-
+        get_dot_htaccess(target)
+    except Exception:
+        cprint("Error""red")
 
 
-
-def get_robots_txt(self, target):
-    cprint("[*]Checking for Robots.txt", 'yellow')
+def get_robots_txt(target):
+    cprint("[*]Checking for Robots.txt"'yellow')
     url = target
     target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
-    req = requests.get(target+"/robots.txt")
-    r = req.text
-    cprint(r, 'blue')
-
-
+    requests.get(target+"/robots.txt")
+    cprint(r'blue')
 
 
-def get_dot_git(self, target):
-    cprint("[*]Checking for .git folder", 'yellow')
+def get_dot_git(target):
+    cprint("[*]Checking for .git folder"'yellow')
     url = target
     target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
     req = requests.get(target+"/.git/")
     r = req.status_code
     if r == 200:
-        cprint("Alert!", 'red')
-        cprint(".git folder exposed publicly", 'red')
+        cprint("Alert!"'red')
+        cprint(".git folder exposed publicly"'red')
     else:
-        print("NO .git folder found", 'blue')
+        print("NO .git folder found"'blue')
 
 
-
-
-def get_dot_svn(self, target):
-    cprint("[*]Checking for .svn folder", 'yellow')
+def get_dot_svn(target):
+    cprint("[*]Checking for .svn folder"'yellow')
     url = target
     target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
     req = requests.get(target+"/.svn/entries")
     r = req.status_code
     if r == 200:
-        cprint("Alert!", 'red')
-        cprint(".SVN folder exposed publicly", 'red')
+        cprint("Alert!"'red')
+        cprint(".SVN folder exposed publicly"'red')
     else:
-        cprint("NO .SVN folder found", 'blue')
+        cprint("NO .SVN folder found"'blue')
+
 
-def get_dot_htaccess(self, target):
-    cprint("[*]Checking for .htaccess", 'yellow')
+def get_dot_htaccess(target):
+    cprint("[*]Checking for .htaccess"'yellow')
     url = target
     target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
     req = requests.get(target+"/.htaccess")
-    r = req.text
     statcode = req.status_code
     if statcode == 403:
-        cprint("403 Forbidden", 'blue')
+        cprint("403 Forbidden"'blue')
     elif statcode == 200:
-        cprint("Alert!!", 'blue')
-        cprint(".htaccess file found!", 'blue')
+        cprint("Alert!!"'blue')
+        cprint(".htaccess file found!"'blue')
     else:
-        cprint("Status code", 'blue')
-        cprint(statcode, 'blue')
+        cprint("Status code"'blue')
+        cprint(statcode)

modules/pagereader.py

@@ -1,28 +1,29 @@
-import urllib.request 
+import urllib.request
 from bs4 import BeautifulSoup
 from modules.bcolors import Bcolors
 
-__all__ = ['readPage']
-
-def readPage(site,printIP=0):
-
- headers = {'User-Agent': 'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot' }
- req = urllib.request.Request(site,None,headers)
-
- try:
-  response = urllib.request.urlopen(req)
-  page = BeautifulSoup(response.read(), 'html.parser')
-
-  if printIP:
-   pg = page.find('strong')
-   IP = pg.renderContents()
-   print(Bcolors.WARNING + Bcolors.BOLD + IP.decode("utf-8") + Bcolors.ENDC)
- except urllib.error.URLError:
-  pass
-
- return page
 
+def readPage(site, printIP=0):
+    b_colors = Bcolors()
+    headers = {'User-Agent':
+               'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot'}
+    req = urllib.request.Request(site, None, headers)
+    attempts_left = 3
 
+    while (attempts_left):
 
+        try:
+            response = urllib.request.urlopen(req)
+            page = BeautifulSoup(response.read(), 'html.parser')
+            if printIP:
+                pg = page.find('strong')
+                IP = pg.renderContents()
+                print(b_colors.WARNING + b_colors.BOLD + IP.decode("utf-8") +
+                      b_colors.ENDC)
+            return page
 
+        except Exception as e:
+            attempts_left -= 1
+            error = e
 
+    raise error