update something

README.md

@@ -92,6 +92,8 @@
 
 - 12-爱企查查询(对外投资，控股公司，分支架构，备案)
 
+这个默认没有在 -d 参数中进行跑，需要手动运行如下命令:
+
 `python3 batch.py -d result.com -cn 横戈信息安全有限公司`
 
 ![company](img/company1.png)
@@ -222,9 +224,9 @@
 
 2、感谢ske大师兄和其他人的项目，前人栽树，后人乘凉
 
-3、虽然说是造轮子，但是对于自己来说还是有收获的
+3、虽然说是造轮子，对于自己来说还是有收获的
 
-#参考文章：
+#参考：
 
 1、https://xz.aliyun.com/t/9508
 
@@ -248,6 +250,8 @@
 
 11、https://github.com/LandGrey/domainNamePredictor
 
+12、https://github.com/sqlmapproject/sqlmap
+
 #需要增加的
 
 ~~1、基于请求数据的时候实现进度可视化，比如进度条~~（已实现）
@@ -348,7 +352,7 @@ EOFError
 
 ~~13、github项目 Common 和 Exploit 和 Spider 都可以去掉~~（已完成）
 
-16、SQL注入自动化探测
+16、SQL注入自动化探测（正在写）
 
 - sql相似度匹配 参考文章：http://mp.weixin.qq.com/s?__biz=Mzg4MzY3MTgyMw==&mid=2247483720&idx=1&sn=5449ed47b74cf892c01eb8833b59c952&chksm=cf429728f8351e3eee7387ca85c79a705ae68122509484d49bc278e24c9de4e22ef0080dc0c8&mpshare=1&scene=23&srcid=1114LGKgJqRAT9xqFA9s2BwC&sharer_sharetime=1636911890316&sharer_shareid=1b35adb1b046ef1a6379932d3eabbaf8#rd
 
@@ -379,3 +383,9 @@ EOFError
 ~~- 添加censys接口~~
 
 ~~- 添加hunter奇安信接口~~
+
+2021.11.25 - 2021.12.9 这两个星期都不更新了，有个证书的考试需要准备下，后面继续改
+
+25、filterCDN方法添加(为后面的portscan节省时间，如果的cdn网段的ip进行端口扫描的话是无意义的)
+
+26、flushIpSegment方法修改（原本清洗数据时间太长，这个方法改了可以缩短清洗数据的时间）

batch.py

@@ -3,11 +3,11 @@
 # @blog     : https://www.cnblogs.com/zpchcbd/
 # @Time     : 2020-11-23 20:45
 
-from core.MyModuleLoader import ModuleLoader
+from core.module.moduleloader import ModuleLoader
 from core.MyConstant import ModulePath
-from core.utils.FuzzDifflib import MyDifflib
+from core.utils.differ import DifferentChecker
 from core.utils.PortWrapper import PortWrapper
-from core.MyLogger import Logger
+from core.log.logger import Logger
 from core.MyGlobalVariableManager import GlobalVariableManager
 
 from spider.BeianSpider import BeianSpider
@@ -431,7 +431,7 @@ def getSimilarityMatch(domain, domainList):
             domainIndex = 0
             while domainIndex < len(newDomainList):
                 current = newDomainList[domainIndex]
-                goodIndexList = MyDifflib.getCloseMatchIndex(current, newDomainList, n=10000, cutoff=0.8)
+                goodIndexList = DifferentChecker.getCloseMatchIndex(current, newDomainList, n=10000, cutoff=0.8)
                 currentResultList = []
                 for index in reversed(sorted(goodIndexList)):
                     currentResultList.append(newDomainList[index])
@@ -455,7 +455,7 @@ def getSimilarityMatch(domain, domainList):
         # -----------------------
 
         # 0、备案查询
-        # self.beianSpider()
+        self.beianSpider()
 
         # 1、checkCdn
         # checkCdn(self.domain)
@@ -469,14 +469,14 @@ def getSimilarityMatch(domain, domainList):
         # self.ksubdomainSpider()
 
         # 3、第三方接口查询
-        # self.thirdSpider()
+        self.thirdSpider()
 
         # 4、SSL/engine/netSpace/github查询
-        # self.threadList.append(Thread(target=self.baiduSpider, ))
-        # self.threadList.append(Thread(target=self.bingSpider, ))
-        # self.threadList.append(Thread(target=self.ctfrSpider, ))
+        self.threadList.append(Thread(target=self.baiduSpider, ))
+        self.threadList.append(Thread(target=self.bingSpider, ))
+        self.threadList.append(Thread(target=self.ctfrSpider, ))
         self.threadList.append(Thread(target=self.netSpider, ))
-        # self.threadList.append(Thread(target=self.githubSpider, ))
+        self.threadList.append(Thread(target=self.githubSpider, ))
         for _ in self.threadList:
             _.start()
         for _ in self.threadList:
@@ -486,38 +486,38 @@ def getSimilarityMatch(domain, domainList):
         # self.flushResult()
 
         # 6、友链爬取
-        # self.friendChainsSpider()
+        self.friendChainsSpider()
 
         # 7、domain2ip
-        # self.domain2ip()
+        self.domain2ip()
 
         # 8、ip2domain
-        # self.ip2domain()
+        self.ip2domain()
 
         # 9、sslSpider @keefe @行牛 @ske 2021.09.01 SSL
         # self.sslSpider()
 
         # 10、alive
-        # self.aliveSpider()
+        self.aliveSpider()
 
         # 11、asn和ip段整理
-        # flushIpSegment(self.domain, self.ipList, self.ipSegmentList)
-        # flushAsn(self.domain, self.asnList)
+        flushIpSegment(self.domain, self.ipList, self.ipSegmentList)
+        flushAsn(self.domain, self.asnList)
 
         # 12、过滤属于CDN网段的IP
         # filterCDN()
 
         # 13、port scan in self.ipPortList
         # print('portConfig: ', portConfig)
-        # portConfig = GlobalVariableManager.getValue('portConfig')
-        # PortWrapper.generatePorts(portConfig, self.ipPortList)
-        # self.ipPortSpider()
+        portConfig = GlobalVariableManager.getValue('portConfig')
+        PortWrapper.generatePorts(portConfig, self.ipPortList)
+        self.ipPortSpider()
 
         # 14、去重子域名
         gDomainList = list(set(gDomainList))
 
         # 15、可探测FUZZ收集
-        # getSimilarityMatch(self.domain, gDomainList)
+        getSimilarityMatch(self.domain, gDomainList)
 
         print('==========================')
         gLogger.info('[+] [AsnList] [{}] {}'.format(len(self.asnList), self.asnList))
@@ -537,7 +537,6 @@ def getSimilarityMatch(domain, domainList):
         gLogger.info('[+] [gDomainList] [{}] {}'.format(len(gDomainList), gDomainList))
         print('==========================')
         gLogger.info('[+] [gDomainAliveList] [{}] {}'.format(len(gDomainAliveList), gDomainAliveList))
-        exit(0)
 
 
 # Exploit
@@ -673,6 +672,8 @@ def parse_args():
         else:
             exit('[-] 文件名{}已存在，如果要运行的话需要将该文件{}.xlsx改名或者删除.'.format(args.domain, args.domain))
     if args.cmsscan:
+        fileName = str(int(time.time()))
+        createXlsx(fileName)
         if args.url:
             moduleLoader = ModuleLoader('exploit')
             if args.module is None:
@@ -689,7 +690,7 @@ def parse_args():
                     exit(0)
             loop = asyncio.get_event_loop()
             domainList = [args.url]
-            cmsScan = CmsScan('result.com', domainList, moduleList)
+            cmsScan = CmsScan(fileName, domainList, moduleList)
             loop.run_until_complete(cmsScan.main())
             print("[+] 总花费时间: " + str(time.time() - starttime))
             exit(0)
@@ -714,23 +715,25 @@ def parse_args():
                 exit('[-] Import Error from core.api import MyNetApi error')
             loop = asyncio.get_event_loop()
             domainList = loop.run_until_complete(MyNetApi.fofaSearch(args.fofa))
-            cmsScan = CmsScan('result.com', domainList, moduleList)
+            cmsScan = CmsScan(fileName, domainList, moduleList)
             loop.run_until_complete(cmsScan.main())
             print("[+] 总花费时间: " + str(time.time() - starttime))
             exit(0)
     # servicescan + portscan
     if args.servicescan:
+        fileName = str(int(time.time()))
+        createXlsx(fileName)
         if args.ips:
             ipPortList = PortWrapper.generateFormat(args.ips)
             PortWrapper.generatePorts(args.port, ipPortList)
-            portscan = PortScan('result.com', ipPortList)
+            portscan = PortScan(fileName, ipPortList)
             loop = asyncio.get_event_loop()
             ipPortServiceList, httpList = loop.run_until_complete(portscan.main())
             total = 0
             for targetService in ipPortServiceList:
                 total += len(targetService['ip'])
             pbar = tqdm(total=total, desc="ServiceScan", ncols=100)  # total是总数
-            servicescan = PortServiceScan('result.com', ipPortServiceList, pbar)
+            servicescan = PortServiceScan(fileName, ipPortServiceList, pbar)
             loop = asyncio.get_event_loop()
             loop.run_until_complete(servicescan.main())
             print("[+] 总花费时间: " + str(time.time() - starttime))
@@ -739,16 +742,18 @@ def parse_args():
             exit('[-] 输入要进行服务扫描的IP')
     # 单独端口扫描选择
     if args.ips:
+        fileName = str(int(time.time()))
+        createXlsx(fileName)
         # 生成ipPortList格式
         ipPortList = PortWrapper.generateFormat(args.ips)
         # 对ipPortList中的ip进行对应的端口填充
         PortWrapper.generatePorts(args.port, ipPortList)
-        portscan = PortScan('result.com', ipPortList)
+        portscan = PortScan(fileName, ipPortList)
         loop = asyncio.get_event_loop()
         ipPortServiceList, httpList = loop.run_until_complete(portscan.main())
-        print("==================Service========================")
+        print("==================Service==================")
         gLogger.info(ipPortServiceList)
-        print("==================HTTP========================")
+        print("===================HTTP===================")
         gLogger.info(httpList)
         print("[+] 总花费时间: " + str(time.time() - starttime))
         exit(0)

common/resolve.py

@@ -16,7 +16,7 @@
 
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
-limit_resolve_conn = 50
+limit_resolve_conn = 100
 
 
 def dns_resolver():

common/tools.py

@@ -135,6 +135,11 @@ def getRootdomain(url):
     return f'{val.domain}.{val.suffix}' if val.domain and val.suffix else ''
 
 
+def getSubdomain(url):
+    val = extract(url)
+    return f'{val.subdomain}.{val.domain}.{val.suffix}' if val.subdomain and val.domain and val.suffix else ''
+
+
 # 创建图表
 def createXlsx(target):
     workbook = xlsxwriter.Workbook(target + ".xlsx")

core/MyModuleManager.py

@@ -2,10 +2,11 @@
 # @Author   : zpchcbd HG team
 # @Time     : 2021-09-10 1:06
 
-from core.MyModuleLoader import ModuleLoader
+from core.module.moduleloader import ModuleLoader
 
 
 class ModuleManager(object):
+    """prepare exploit, saving time for multi save module in cmsExploit @zpchcbd"""
     def __init__(self):
         pass
         # self.moduleLoader = ModuleLoader()

core/api/MyNetApi.py

@@ -2,7 +2,7 @@
 # @Author   : zpchcbd HG team
 # @Time     : 2021-09-10 21:02
 import base64
-from core.MyAsyncHttp import *
+from core.request.asynchttp import *
 from spider.common import config
 
 

core/component/MyDict.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+# @Author   : zpchcbd HG team
+# @Time     : 2021-09-06 20:48
+
+class Mydict(dict):
+    def __getattr__(self, item):
+        try:
+            return self.__getitem__(item)
+        except KeyError:
+            raise AttributeError("unable to access item '{}'".format(item))
+
+
+if __name__ == '__main__':
+    a = Mydict()
+    a['a'] = 1
+    print(a['b'])

core/component/MyList.py

@@ -0,0 +1,7 @@
+# coding=utf-8
+# @Author   : zpchcbd HG team
+# @Time     : 2021-09-06 20:49
+
+class Mylist(list):
+    def __getattr__(self, item):
+        pass

core/constant.py

@@ -0,0 +1,13 @@
+# coding=utf-8
+# @Author   : zpchcbd HG team
+# @Time     : 2021-09-10 14:14
+
+
+class ModulePath:
+    EXPLOIT = 'exploit/web/'
+    THIRDLIB = 'spider/thirdLib/'
+
+
+class ProgramPath:
+    PYTHON = 'lib/python.exe'
+    SQLMAP = 'lib/sqlmap/'

core/log/logger.py

@@ -0,0 +1,59 @@
+# coding=utf-8
+# @Author   : zpchcbd HG team
+# @blog     : https://www.cnblogs.com/zpchcbd/
+# @Time     : 2021-11-22 12:59
+
+"""封装一个日志类，想要实现的是文件和控制台都可以记录相关信息"""
+import logging
+
+
+class Logger:
+    def __init__(self, path, clevel=logging.DEBUG, Flevel=logging.DEBUG):
+        self.logger = logging.getLogger(path)
+        self.logger.setLevel(logging.DEBUG)  # 设置logger级别
+        self.formatter = logging.Formatter('[%(levelname)s]%(asctime)s %(message)s')
+
+        sh = logging.StreamHandler()
+        sh.setFormatter(self.formatter)
+        sh.setLevel(clevel)  # 设置处理器的Level
+
+        fh = logging.FileHandler(path)
+        fh.setFormatter(self.formatter)
+        fh.setLevel(Flevel)  # 设置处理器的Level
+
+        self.logger.addHandler(sh)
+        self.logger.addHandler(fh)
+
+    def getLogger(self):
+        return self.logger
+
+    def debug(self, message):
+        self.logger.debug(message)
+
+    def info(self, message):
+        self.logger.info(message)
+
+    def warn(self, message):
+        self.logger.warning(message)
+
+    def error(self, message):
+        self.logger.error(message)
+
+    def critical(self, message):
+        self.logger.critical(message)
+
+
+if __name__ == '__main__':
+    mLogger = Logger('./logs.txt', logging.DEBUG, logging.DEBUG)
+    # mLogger.debug('HengGe test...., , debug')
+    # mLogger.info('HengGe test...., , info')
+    # mLogger.warn('HengGe test....,  warning')
+    # mLogger.error('HengGe test....,  error')
+    # mLogger.cri('HengGe test...., , critical')
+    asnList = [{'service': 'http', 'ip': ['47.110.217.169:8080', '47.113.23.213:8080', '58.251.27.73:8080', '113.98.59.166:8080', '63.221.140.244:8080', '47.254.137.137:8080', '58.251.27.73:9000']}, {'service': 'bgp', 'ip': ['58.60.230.102:179']}, {'service': 'https-alt', 'ip': ['47.110.217.169:8443', '47.96.196.50:8443']}, {'service': 'osiris', 'ip': ['103.27.119.242:541']}, {'service': 'cisco-sccp', 'ip': ['58.60.230.103:2000']}, {'service': 'redis', 'ip': ['127.0.0.1:6377']}, {'service': 'smtp', 'ip': ['202.103.147.169:25', '202.103.147.161:25', '63.217.80.70:25', '202.103.147.172:25']}, {'service': 'ssl/http', 'ip': ['47.52.122.123:8443']}, {'service': 'http-proxy', 'ip': ['222.134.66.173:8080', '222.134.66.177:8080']}]
+    ip = [{'ipSegment': '183.232.187.0/24', 'ip': ['183.232.187.210', '183.232.187.201', '183.232.187.197'], 'num': 3}, {'ipSegment': '218.2.178.0/24', 'ip': ['218.2.178.29', '218.2.178.22', '218.2.178.23', '218.2.178.21', '218.2.178.15', '218.2.178.14', '218.2.178.27', '218.2.178.32'], 'num': 8}]
+    mLogger.info('111111')
+    mLogger.info('111111')
+    mLogger.info('111111')
+    mLogger.info('222222')
+    mLogger.info('222222')