diff --git a/HackRequests/HackRequests.py b/HackRequests/HackRequests.py index 8706098..53b09d3 100644 --- a/HackRequests/HackRequests.py +++ b/HackRequests/HackRequests.py @@ -14,6 +14,9 @@ import zlib from http import client from urllib import parse +import re +import time +from datetime import timedelta class HackError(Exception): @@ -101,6 +104,48 @@ def _make_con(self, scheme, host, port, proxy=None): raise Exception('connect err') +# copy from requests +UNRESERVED_CHARS = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" +) +SUB_DELIM_CHARS = set("!$&'()*+,;=") +USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} +PATH_CHARS = USERINFO_CHARS | {"@", "/"} +PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") +def encode_invalid_chars(component, allowed_chars, encoding="utf-8"): + """Percent-encodes a URI component without reapplying + onto an already percent-encoded component. + """ + if component is None: + return component + + # component = six.ensure_text(component) + + # Normalize existing percent-encoded bytes. + # Try to see if the component we're encoding is already percent-encoded + # so we can skip all '%' characters but still encode all others. + component, percent_encodings = PERCENT_RE.subn( + lambda match: match.group(0).upper(), component + ) + + uri_bytes = component.encode("utf-8", "surrogatepass") + is_percent_encoded = percent_encodings == uri_bytes.count(b"%") + encoded_component = bytearray() + + for i in range(0, len(uri_bytes)): + # Will return a single character bytestring on both Python 2 & 3 + byte = uri_bytes[i : i + 1] + byte_ord = ord(byte) + if (is_percent_encoded and byte == b"%") or ( + byte_ord < 128 and byte.decode() in allowed_chars + ): + encoded_component += byte + continue + encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) + + return encoded_component.decode(encoding) + + class hackRequests(object): ''' hackRequests是主要http请求函数。 @@ -108,15 +153,15 @@ class hackRequests(object): 可以通过http或者httpraw来访问网络 ''' - def __init__(self, conpool=None): + def __init__(self, conpool=None,timeout=17): self.lock = threading.Lock() if conpool is None: - self.httpcon = httpcon(timeout=17) + self.httpcon = httpcon(timeout=timeout) else: self.httpcon = conpool - def _get_urlinfo(self, url, realhost: str): + def _get_urlinfo(self, url, params: dict,realhost: str): p = parse.urlparse(url) scheme = p.scheme.lower() if scheme != "http" and scheme != "https": @@ -125,11 +170,17 @@ def _get_urlinfo(self, url, realhost: str): port = 80 if scheme == "http" else 443 if ":" in hostname: hostname, port = hostname.split(":") - path = "" - if p.path: + query = parse.urlencode(params) + if p.path == "": + path = "/" + else: path = p.path - if p.query: - path = path + "?" + p.query + if p.query: + path = path + "?" + encode_invalid_chars(p.query,PATH_CHARS) + if query: + path = path + "&" + query + elif query: + path = path + "?" + query if realhost: if ":" not in realhost: realhost = realhost + ":80" @@ -144,7 +195,7 @@ def _send_output_hook(*args, **kwargs): return _send_output_hook - def httpraw(self, raw: str, **kwargs): + def httpraw(self, raw: bytes, **kwargs): raw = raw.strip() proxy = kwargs.get("proxy", None) real_host = kwargs.get("real_host", None) @@ -158,19 +209,19 @@ def httpraw(self, raw: str, **kwargs): port = 443 try: - index = raw.index('\n') + index = raw.index(b'\n') except ValueError: raise Exception("ValueError") log = {} try: - method, path, protocol = raw[:index].split(" ") + method, path, protocol = raw[:index].decode().split(" ") except: raise Exception("Protocol format error") raw = raw[index + 1:] try: - host_start = raw.index("Host: ") - host_end = raw.index('\n', host_start) + host_start = raw.index(b"Host: ") + host_end = raw.index(b'\n', host_start) except ValueError: raise ValueError("Host headers not found") @@ -180,7 +231,7 @@ def httpraw(self, raw: str, **kwargs): if ":" in real_host: host, port = real_host.split(":") else: - host = raw[host_start + len("Host: "):host_end] + host = raw[host_start + len(b"Host: "):host_end].decode() if ":" in host: host, port = host.split(":") raws = raw.splitlines() @@ -193,20 +244,22 @@ def httpraw(self, raw: str, **kwargs): index = 0 for r in raws: - if r == "": + if r == b"": break try: - k, v = r.split(": ") + k, v = r.split(b": ") except: k = r - v = "" + v = b"" headers[k] = v index += 1 - headers["Connection"] = "close" + headers[b"Connection"] = b"close" + if b"Content-Length" in headers: + headers.pop(b"Content-Length") if len(raws) < index + 1: - body = '' + body = b'' else: - body = '\n'.join(raws[index + 1:]).lstrip() + body = b'\n'.join(raws[index + 1:]).lstrip() urlinfo = scheme, host, int(port), path @@ -219,17 +272,17 @@ def httpraw(self, raw: str, **kwargs): conn.putrequest(method, path, skip_host=True, skip_accept_encoding=True) for k, v in headers.items(): conn.putheader(k, v) - if body and "Content-Length" not in headers and "Transfer-Encoding" not in headers: + if body and b"Content-Length" not in headers and b"Transfer-Encoding" not in headers: length = conn._get_content_length(body, method) - conn.putheader("Content-Length", length) + conn.putheader(b"Content-Length", length) conn.endheaders() if body: - if headers.get("Transfer-Encoding", '').lower() == "chunked": - body = body.replace('\r\n', '\n') - body = body.replace('\n', '\r\n') - body = body + "\r\n" * 2 - log["request"] += "\r\n" + body - conn.send(body.encode('utf-8')) + if headers.get(b"Transfer-Encoding", b'').lower() == b"chunked": + body = body.replace(b'\r\n', b'\n') + body = body.replace(b'\n', b'\r\n') + body = body + b"\r\n" * 2 + log["request"] += "\r\n" + body.decode() + conn.send(body) rep = conn.getresponse() except socket.timeout: raise HackError("socket connect timeout") @@ -246,13 +299,12 @@ def httpraw(self, raw: str, **kwargs): _url = "{scheme}://{host}{path}".format(scheme=scheme, host=host, path=path) else: _url = "{scheme}://{host}{path}".format(scheme=scheme, host=host + ":" + port, path=path) - + redirect = rep.msg.get('location', None) # handle 301/302 if redirect and location: if not redirect.startswith('http'): redirect = parse.urljoin(_url, redirect) return self.http(redirect, post=None, method=method, headers=headers, location=True, locationcount=1) - return response(rep, _url, log, ) def http(self, url, **kwargs): @@ -263,7 +315,7 @@ def http(self, url, **kwargs): proxy = kwargs.get('proxy', None) headers = kwargs.get('headers', {}) - + params = kwargs.get("params", {}) # real host:ip real_host = kwargs.get("real_host", None) @@ -286,7 +338,8 @@ def http(self, url, **kwargs): if "Content-Length" in headers: del headers["Content-Length"] - urlinfo = scheme, host, port, path = self._get_urlinfo(url, real_host) + urlinfo = scheme, host, port, path = self._get_urlinfo(url, params,real_host) + log = {} try: conn = self.httpcon.get_con(urlinfo, proxy=proxy) @@ -297,17 +350,18 @@ def http(self, url, **kwargs): if post: method = "POST" if isinstance(post, str): + post = encode_invalid_chars(post) + elif isinstance(post,dict): try: - post = extract_dict(post, sep="&") + post = parse.urlencode(post) except: pass - try: - post = parse.urlencode(post) - except: - pass + else: + raise Exception("post must be str or dict") if "Content-Type" not in headers: tmp_headers["Content-Type"] = kwargs.get( - "Content-type", "application/json") + "Content-type", "application/x-www-form-urlencoded") + if 'Accept' not in headers: tmp_headers["Accept"] = tmp_headers.get("Accept", "*/*") if 'Accept-Encoding' not in headers: @@ -319,8 +373,10 @@ def http(self, url, **kwargs): 'User-Agent') else 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' try: + start = time.perf_counter() conn.request(method, path, post, tmp_headers) rep = conn.getresponse() + elapsed = time.perf_counter() - start # body = rep.read() except socket.timeout: raise HackError("socket connect timeout") @@ -347,16 +403,17 @@ def http(self, url, **kwargs): if not redirect: redirect = url log["url"] = redirect - return response(rep, redirect, log, cookie) + return response(rep, redirect, log, cookie,timedelta(seconds=elapsed)) class response(object): - def __init__(self, rep, redirect, log, oldcookie=''): + def __init__(self, rep, redirect, log, oldcookie='',elapsed = None): self.rep = rep self.status_code = self.rep.status # response code self.url = redirect self._content = b'' + self.elapsed = elapsed _header_dict = dict() self.cookie = "" @@ -462,6 +519,9 @@ def __init__(self, threadnum, callback, timeout=10): self.isContinue = True self.thread_count_lock = threading.Lock() self._callback = callback + + def set_callback(self,callback): + self._callback = callback def push(self, payload): self.queue.put(payload) @@ -496,10 +556,13 @@ def http(self, url, **kwargs): func = self.hack.http self.queue.put({"func": func, "url": url, "kw": kwargs}) - def httpraw(self, raw: str, ssl: bool = False, proxy=None, location=True): + def httpraw(self, raw: str, ssl: bool = False, proxy=None, location=True,real_host=None): func = self.hack.httpraw - self.queue.put({"func": func, "raw": raw, "ssl": ssl, - "proxy": proxy, "location": location}) + params = {"func": func, "raw": raw, "ssl": ssl, + "proxy": proxy, "location": location} + if real_host != None: + params.update({"real_host":real_host}) + self.queue.put(params) def scan(self): while 1: @@ -523,16 +586,16 @@ def scan(self): def http(url, **kwargs): - # timeout = kwargs.get("timeout", 10) + timeout = kwargs.get("timeout", 17) # con = httpcon(timeout=timeout) - hack = hackRequests() + hack = hackRequests(timeout=timeout) return hack.http(url, **kwargs) def httpraw(raw: str, **kwargs): - # con = httpcon(timeout=timeout) + timeout = kwargs.get("timeout", 17) # hack = hackRequests(con) - hack = hackRequests() + hack = hackRequests(timeout=timeout) return hack.httpraw(raw, **kwargs) diff --git a/README.md b/README.md index 7475dc6..6f6db0e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ +# 魔改增加功能： + +1. 为 `hackRequests.http()` / `http()` 增加了个 `params` 参数，支持传入 dict 形式的 query string（像 requests 一样） +2. 为 `hackRequests.http()` / `http()` 增加了个 `timeout` 参数 ，这样写时间盲注之类的脚本的时候会方便一点 +3. 增加了个 `encode_invalid_chars()` 函数，这样 url 当中的非法字符就会被 urlencode +4. 给 `response` 类增加了一个 `elapsed` ，通过调用 `resp.elapsed.total_seconds()` 能获得一次请求所用的时间，主要也是为了写盲注之类的脚本方便 + # hack-requests + HackRequests 是基于`Python3.x`的一个给黑客们使用的http底层网络库。如果你需要一个不那么臃肿而且像requests一样优雅的设计，并且提供底层请求包/返回包原文来方便你进行下一步分析，如果你使用Burp Suite，可以将原始报文直接复制重放，对于大量的HTTP请求，hack-requests线程池也能帮你实现最快速的响应。 - 像requests一样好用的设计 @@ -245,4 +253,3 @@ threadpool.run() | stop() | | 停止线程池 | | run() | | 启动线程池 | -