refactor + misc improvements (#499)

* refactor + misc improvements * support for http://proxify vhost * feat: yaml multidoc writer * fix lint error * update docs * fix -sr empty file issue * typo fix * show warnings instead of errors * fix missing body issue * implement filter/match in jsonl/yaml * add more request variables * you new yaml library + multiline strings * fix content length + body length issue * fix boolean logic error
projectdiscovery · Feb 23, 2024 · 3fb0424 · 3fb0424
1 parent 15fefa0
commit 3fb0424
Show file tree

Hide file tree

Showing 16 changed files with 717 additions and 388 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,8 @@ cmd/mitmrelay/mitmrelay
 dist/*
 
 .vscode
-.devcontainer
+.devcontainer
+**/proxify_logs.jsonl
+**/proxify_logs.yaml
+/proxify
+**/proxify_logs
diff --git a/README.md b/README.md
@@ -1,9 +1,7 @@
 <h1 align="center">
-  <img src="static/proxify-logo.png" alt="proxify" width="200px">
-  <br>
+  <img src="static/proxify-logo.png" alt="proxify" width="200px" />
 </h1>
 
-
 <p align="center">
 <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/license-MIT-_red.svg"></a>
 <a href="https://github.com/projectdiscovery/proxify/issues"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat"></a>
@@ -61,17 +59,20 @@ proxify -h
 
 This will display help for the tool. Here are all the switches it supports.
 
-```shell
+```console
+Swiss Army Knife Proxy for rapid deployments. Supports multiple operations such as request/response dump,filtering and manipulation via DSL language, upstream HTTP/Socks5 proxy
+
 Usage:
   ./proxify [flags]
 
 Flags:
 OUTPUT:
-   -o, -output string    Output Directory to store HTTP proxy logs (default "logs")
-   -dump-req             Dump only HTTP requests to output file
-   -dump-resp            Dump only HTTP responses to output file
-   -j, -jsonl            write output in JSONL(ines) format
-   -oca, -out-ca string  Generate and Save CA File to filename
+   -sr, -store-resposne        store raw http request / response to output directory (default proxify_logs)
+   -o, -output                 output file to store proxify logs (default proxify_logs.jsonl)
+   -of, -output-format string  output format (jsonl/yaml) (default "jsonl")
+   -dump-req                   Dump only HTTP requests to output file
+   -dump-resp                  Dump only HTTP responses to output file
+   -oca, -out-ca string        Generate and Save CA File to filename
 
 UPDATE:
    -up, -update                 update proxify to latest version
@@ -85,7 +86,7 @@ FILTER:
 
 NETWORK:
    -ha, -http-addr string    Listening HTTP IP and Port address (ip:port) (default "127.0.0.1:8888")
-   -sa, -socks-addr string   Listening SOCKS IP and Port address (ip:port) (default "127.0.0.1:10080")
+   -sa, -socks-addr          Listening SOCKS IP and Port address (ip:port) (default 127.0.0.1:10080)
    -da, -dns-addr string     Listening DNS IP and Port address (ip:port)
    -dm, -dns-mapping string  Domain to IP DNS mapping (eg domain:ip,domain:ip,..)
    -r, -resolver string      Custom DNS resolvers to use (ip:port)
@@ -100,19 +101,20 @@ EXPORT:
 
 CONFIGURATION:
    -config string              path to the proxify configuration file
-   -ec, -export-config string  proxify export module configuration file ($HOME/.config/proxify/export-config.yaml)
-   -config-directory string    override the default config path ($HOME/.config/proxify)
+   -ec, -export-config string  proxify export module configuration file (default "$CONFIG/export-config.yaml")
+   -config-directory string    override the default config path (default "$CONFIG/proxify")
    -cert-cache-size int        Number of certificates to cache (default 256)
    -a, -allow string[]         Allowed list of IP/CIDR's to be proxied
    -d, -deny string[]          Denied list of IP/CIDR's to be proxied
    -pt, -passthrough string[]  List of passthrough domains
 
 DEBUG:
-   -nc, -no-color      No Color (default true)
+   -nc, -no-color      No Color
    -version            Version
    -silent             Silent
    -v, -verbose        Verbose
    -vv, -very-verbose  Very Verbose
+
 ```
 
 ### Running Proxify
@@ -152,14 +154,57 @@ proxify -socks5-proxy 127.0.0.1:9050
 
 ### Dump all the HTTP/HTTPS traffic
 
-Dump all the traffic into separate files with request followed by the response:
+Proxify supports three output formats: **JSONL**, **YAML** and **Files**.
 
-```shell
-proxify -output logs
+**JSONL** (default):
+
+In Json Lines format each Http Request/Response pair is stored as json object in a single line. 
+
+```json
+{"timestamp":"2024-02-20T01:56:49+05:30","url":"https://scanme.sh:443","request":{"header":{"Connection":"close","User-Agent":"curl/8.1.2","host":"scanme.sh:443","method":"CONNECT","path":"","scheme":"https"},"raw":"CONNECT scanme.sh:443 HTTP/1.1\r\nHost: scanme.sh:443\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"},"response":{"header":{"Content-Length":"0"},"raw":"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"}}
+{"timestamp":"2024-02-20T01:56:49+05:30","url":"https://scanme.sh/","request":{"header":{"Accept":"*/*","Connection":"close","User-Agent":"curl/8.1.2","host":"scanme.sh","method":"GET","path":"/","scheme":"https"},"raw":"GET / HTTP/1.1\r\nHost: scanme.sh\r\nAccept: */*\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"},"response":{"header":{"Content-Type":"text/plain; charset=utf-8","Date":"Mon, 19 Feb 2024 20:26:49 GMT"},"body":"ok","raw":"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Mon, 19 Feb 2024 20:26:49 GMT\r\n\r\n"}}
 ```
 
-As default, proxied requests/responses are stored in the **logs** folder. Additionally, **dump-req** or **dump-resp** flag can be used for saving specific part of the request to the file.
+**Yaml MultiDoc**:
+
+In the YAML MultiDoc format, each HTTP request and response pair is encapsulated as a separate document.All Documents in output yaml file are seperated  by `---` to allow stream parsing and consumption.
+
+```console
+proxify -output-format yaml
+```
+
+```yaml
+timestamp: "2024-02-20T01:40:40+05:30"
+url: https://scanme.sh:443
+request:
+    header:
+        Connection: close
+        User-Agent: curl/8.1.2
+        host: scanme.sh:443
+        method: CONNECT
+        path: ""
+        scheme: https
+    body: ""
+    raw: "CONNECT scanme.sh:443 HTTP/1.1\r\nHost: scanme.sh:443\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"
+response:
+    header:
+        Content-Length: "0"
+    body: ""
+    raw: "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"
+---
+timestamp: "2024-02-20T01:40:40+05:30"
+...
+```
+
+**Files**:
+
+In Files format, each HTTP request and response pair is stored in separate files with the request followed by the response. Filenames are in format of `{{Host}}-{{randstr}}.txt`. Additionally, **dump-req** or **dump-resp** flag can be used for saving specific part of the request to the file.
+
+```console
+proxify -store-response
+```
 
+>Note: When using `-store-response` both jsonl and files are generated.
 
 ### Hostname mapping with Local DNS resolver
 

diff --git a/go.mod b/go.mod
@@ -9,24 +9,26 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/elastic/go-elasticsearch/v7 v7.17.10
 	github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
+	github.com/goccy/go-yaml v1.11.3
 	github.com/haxii/fastproxy v0.5.37
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/dsl v0.0.43
 	github.com/projectdiscovery/fastdialer v0.0.59
 	github.com/projectdiscovery/goflags v0.1.39
 	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/martian/v3 v3.0.0-20230412114616-98e3a0a6994a
+	github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477
 	github.com/projectdiscovery/roundrobin v0.0.6
 	github.com/projectdiscovery/tinydns v0.0.29
-	github.com/projectdiscovery/utils v0.0.79
-	golang.org/x/net v0.17.0
+	github.com/projectdiscovery/utils v0.0.80-0.20240219143814-1bd72bb71244
+	golang.org/x/net v0.21.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/mholt/archiver/v3 v3.5.1 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 )
 
 require (
@@ -36,7 +38,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/charmbracelet/glamour v0.6.0 // indirect
@@ -52,13 +54,13 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20210110162100-a92cc753f88e // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/gaukas/godicttls v0.0.4 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
@@ -71,13 +73,13 @@ require (
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kataras/jwt v0.1.8 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.25 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
 	github.com/miekg/dns v1.1.56 // indirect
 	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -86,7 +88,7 @@ require (
 	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect
 	github.com/projectdiscovery/hmap v0.0.39 // indirect
@@ -121,11 +123,11 @@ require (
 	github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect