- +
-
@@ -61,17 +59,20 @@ proxify -h
This will display help for the tool. Here are all the switches it supports.
-```shell
+```console
+Swiss Army Knife Proxy for rapid deployments. Supports multiple operations such as request/response dump,filtering and manipulation via DSL language, upstream HTTP/Socks5 proxy
+
Usage:
./proxify [flags]
Flags:
OUTPUT:
- -o, -output string Output Directory to store HTTP proxy logs (default "logs")
- -dump-req Dump only HTTP requests to output file
- -dump-resp Dump only HTTP responses to output file
- -j, -jsonl write output in JSONL(ines) format
- -oca, -out-ca string Generate and Save CA File to filename
+ -sr, -store-resposne store raw http request / response to output directory (default proxify_logs)
+ -o, -output output file to store proxify logs (default proxify_logs.jsonl)
+ -of, -output-format string output format (jsonl/yaml) (default "jsonl")
+ -dump-req Dump only HTTP requests to output file
+ -dump-resp Dump only HTTP responses to output file
+ -oca, -out-ca string Generate and Save CA File to filename
UPDATE:
-up, -update update proxify to latest version
@@ -85,7 +86,7 @@ FILTER:
NETWORK:
-ha, -http-addr string Listening HTTP IP and Port address (ip:port) (default "127.0.0.1:8888")
- -sa, -socks-addr string Listening SOCKS IP and Port address (ip:port) (default "127.0.0.1:10080")
+ -sa, -socks-addr Listening SOCKS IP and Port address (ip:port) (default 127.0.0.1:10080)
-da, -dns-addr string Listening DNS IP and Port address (ip:port)
-dm, -dns-mapping string Domain to IP DNS mapping (eg domain:ip,domain:ip,..)
-r, -resolver string Custom DNS resolvers to use (ip:port)
@@ -100,19 +101,20 @@ EXPORT:
CONFIGURATION:
-config string path to the proxify configuration file
- -ec, -export-config string proxify export module configuration file ($HOME/.config/proxify/export-config.yaml)
- -config-directory string override the default config path ($HOME/.config/proxify)
+ -ec, -export-config string proxify export module configuration file (default "$CONFIG/export-config.yaml")
+ -config-directory string override the default config path (default "$CONFIG/proxify")
-cert-cache-size int Number of certificates to cache (default 256)
-a, -allow string[] Allowed list of IP/CIDR's to be proxied
-d, -deny string[] Denied list of IP/CIDR's to be proxied
-pt, -passthrough string[] List of passthrough domains
DEBUG:
- -nc, -no-color No Color (default true)
+ -nc, -no-color No Color
-version Version
-silent Silent
-v, -verbose Verbose
-vv, -very-verbose Very Verbose
+
```
### Running Proxify
@@ -152,14 +154,57 @@ proxify -socks5-proxy 127.0.0.1:9050
### Dump all the HTTP/HTTPS traffic
-Dump all the traffic into separate files with request followed by the response:
+Proxify supports three output formats: **JSONL**, **YAML** and **Files**.
-```shell
-proxify -output logs
+**JSONL** (default):
+
+In Json Lines format each Http Request/Response pair is stored as json object in a single line.
+
+```json
+{"timestamp":"2024-02-20T01:56:49+05:30","url":"https://scanme.sh:443","request":{"header":{"Connection":"close","User-Agent":"curl/8.1.2","host":"scanme.sh:443","method":"CONNECT","path":"","scheme":"https"},"raw":"CONNECT scanme.sh:443 HTTP/1.1\r\nHost: scanme.sh:443\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"},"response":{"header":{"Content-Length":"0"},"raw":"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"}}
+{"timestamp":"2024-02-20T01:56:49+05:30","url":"https://scanme.sh/","request":{"header":{"Accept":"*/*","Connection":"close","User-Agent":"curl/8.1.2","host":"scanme.sh","method":"GET","path":"/","scheme":"https"},"raw":"GET / HTTP/1.1\r\nHost: scanme.sh\r\nAccept: */*\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"},"response":{"header":{"Content-Type":"text/plain; charset=utf-8","Date":"Mon, 19 Feb 2024 20:26:49 GMT"},"body":"ok","raw":"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Mon, 19 Feb 2024 20:26:49 GMT\r\n\r\n"}}
```
-As default, proxied requests/responses are stored in the **logs** folder. Additionally, **dump-req** or **dump-resp** flag can be used for saving specific part of the request to the file.
+**Yaml MultiDoc**:
+
+In the YAML MultiDoc format, each HTTP request and response pair is encapsulated as a separate document.All Documents in output yaml file are seperated by `---` to allow stream parsing and consumption.
+
+```console
+proxify -output-format yaml
+```
+
+```yaml
+timestamp: "2024-02-20T01:40:40+05:30"
+url: https://scanme.sh:443
+request:
+ header:
+ Connection: close
+ User-Agent: curl/8.1.2
+ host: scanme.sh:443
+ method: CONNECT
+ path: ""
+ scheme: https
+ body: ""
+ raw: "CONNECT scanme.sh:443 HTTP/1.1\r\nHost: scanme.sh:443\r\nConnection: close\r\nUser-Agent: curl/8.1.2\r\n\r\n"
+response:
+ header:
+ Content-Length: "0"
+ body: ""
+ raw: "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"
+---
+timestamp: "2024-02-20T01:40:40+05:30"
+...
+```
+
+**Files**:
+
+In Files format, each HTTP request and response pair is stored in separate files with the request followed by the response. Filenames are in format of `{{Host}}-{{randstr}}.txt`. Additionally, **dump-req** or **dump-resp** flag can be used for saving specific part of the request to the file.
+
+```console
+proxify -store-response
+```
+>Note: When using `-store-response` both jsonl and files are generated.
### Hostname mapping with Local DNS resolver
diff --git a/go.mod b/go.mod
index 35c3e16e..8a57a8c1 100644
--- a/go.mod
+++ b/go.mod
@@ -9,24 +9,26 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
github.com/elastic/go-elasticsearch/v7 v7.17.10
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
+ github.com/goccy/go-yaml v1.11.3
github.com/haxii/fastproxy v0.5.37
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/dsl v0.0.43
github.com/projectdiscovery/fastdialer v0.0.59
github.com/projectdiscovery/goflags v0.1.39
github.com/projectdiscovery/gologger v1.1.12
- github.com/projectdiscovery/martian/v3 v3.0.0-20230412114616-98e3a0a6994a
+ github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477
github.com/projectdiscovery/roundrobin v0.0.6
github.com/projectdiscovery/tinydns v0.0.29
- github.com/projectdiscovery/utils v0.0.79
- golang.org/x/net v0.17.0
+ github.com/projectdiscovery/utils v0.0.80-0.20240219143814-1bd72bb71244
+ golang.org/x/net v0.21.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/docker/go-units v0.5.0 // indirect
- github.com/klauspost/pgzip v1.2.5 // indirect
+ github.com/klauspost/pgzip v1.2.6 // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
+ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
)
require (
@@ -36,7 +38,7 @@ require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
- github.com/andybalholm/brotli v1.0.6 // indirect
+ github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/charmbracelet/glamour v0.6.0 // indirect
@@ -52,13 +54,13 @@ require (
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/elazarl/goproxy/ext v0.0.0-20210110162100-a92cc753f88e // indirect
- github.com/fatih/color v1.15.0 // indirect
+ github.com/fatih/color v1.16.0 // indirect
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
- github.com/gorilla/css v1.0.0 // indirect
+ github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
@@ -71,13 +73,13 @@ require (
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kataras/jwt v0.1.8 // indirect
- github.com/klauspost/compress v1.16.7 // indirect
+ github.com/klauspost/compress v1.17.6 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-isatty v0.0.19 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
- github.com/microcosm-cc/bluemonday v1.0.25 // indirect
+ github.com/microcosm-cc/bluemonday v1.0.26 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -86,7 +88,7 @@ require (
github.com/muesli/termenv v0.15.1 // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
- github.com/pierrec/lz4/v4 v4.1.17 // indirect
+ github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/gostruct v0.0.2 // indirect
github.com/projectdiscovery/hmap v0.0.39 // indirect
@@ -121,11 +123,11 @@ require (
github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/crypto v0.17.0 // indirect
+ golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
- golang.org/x/sys v0.16.0 // indirect
+ golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index 03d4266e..f7b34d49 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,8 @@ github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YT
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
-github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
@@ -79,8 +79,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -90,8 +90,16 @@ github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXb
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -133,8 +141,9 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -173,11 +182,12 @@ github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk=
github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
+github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -186,6 +196,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -193,8 +205,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -202,8 +214,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
-github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg=
-github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=
+github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
+github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
@@ -239,8 +251,8 @@ github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
-github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -261,8 +273,8 @@ github.com/projectdiscovery/hmap v0.0.39 h1:R33JJzPz8TMUm1TJQ/X5zVJbmyTS64EttZ08
github.com/projectdiscovery/hmap v0.0.39/go.mod h1:wEPoEIVGPdHsc9EnE+ERUdgNyp29zZn6gACOALylHOg=
github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
-github.com/projectdiscovery/martian/v3 v3.0.0-20230412114616-98e3a0a6994a h1:TpQ/kMGhsFIv/9hytTU+xXaL+yl2xnTkxLqQRlV6+zE=
-github.com/projectdiscovery/martian/v3 v3.0.0-20230412114616-98e3a0a6994a/go.mod h1:wPvVUl2C/XOFacugXwsUp65GN0upmKfwKMyimA/AAaM=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477 h1:VJaBELAC5Hw+kc+ylrFF5nSf7Wasnb9mZxMlF/kR3gg=
+github.com/projectdiscovery/martian/v3 v3.0.0-20240219194442-fed3b744f477/go.mod h1:wPvVUl2C/XOFacugXwsUp65GN0upmKfwKMyimA/AAaM=
github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw+NFp/4qIK688x7o=
github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI=
github.com/projectdiscovery/retryabledns v1.0.56 h1:Rk/fvBSNjw4vzbHRSSoFz3Bkn9uaRSk0UE/IsEBl0cQ=
@@ -271,8 +283,8 @@ github.com/projectdiscovery/roundrobin v0.0.6 h1:zoJAFRgP9XK7B+iKSjR+djRAuDYxnc5
github.com/projectdiscovery/roundrobin v0.0.6/go.mod h1:vTxcWqNLyMH6VE2Q/hsNNvDHFLiIzHozC1rLLT/vocQ=
github.com/projectdiscovery/tinydns v0.0.29 h1:0xBFo+jJcEpf/zlYtxD4fQNeqrDvhzBrOfmpXdKKJKY=
github.com/projectdiscovery/tinydns v0.0.29/go.mod h1:ic2bJKVMexcpxeBTtO1klR2sddmk4jV6+WDZptr1VxU=
-github.com/projectdiscovery/utils v0.0.79 h1:ptO3Qo2e24SK5w5yvDk2whsvSEIk7gSX+RNhBQPRKqc=
-github.com/projectdiscovery/utils v0.0.79/go.mod h1:tBFlI+1warN7y7hKpFf6pqqOszvufENofy9Md0qlZQo=
+github.com/projectdiscovery/utils v0.0.80-0.20240219143814-1bd72bb71244 h1:vqMyenRxz07UC5m+uvRdI+18fBw96+wD6JAJizy9up8=
+github.com/projectdiscovery/utils v0.0.80-0.20240219143814-1bd72bb71244/go.mod h1:avG/w3yO6v/P2jY/rjfjW8mByRbpY+lEBO8wASCh9yM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
@@ -382,8 +394,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
@@ -418,8 +430,8 @@ golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
@@ -457,8 +469,8 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -492,6 +504,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
diff --git a/internal/runner/options.go b/internal/runner/options.go
index 2e18cce2..ab746c8f 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -4,6 +4,7 @@ import (
"math"
"os"
"path/filepath"
+ "strings"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
@@ -28,6 +29,7 @@ var (
type Options struct {
OutputDirectory string
OutputFile string // for storing the jsonl output
+ OutputFormat string
LoggerConfig string
ConfigDir string
CertCacheSize int
@@ -72,11 +74,11 @@ func ParseOptions() (*Options, error) {
flagSet.CreateGroup("output", "Output",
// Todo: flagSet.BoolVar(&options.Dump, "dump", true, "Dump HTTP requests/response to output file"),
- flagSet.DynamicVarP(&options.OutputDirectory, "store-resposne", "sr", "proxify_logs", "store raw http request / response to output directory"),
+ flagSet.DynamicVarP(&options.OutputDirectory, "store-response", "sr", "proxify_logs", "store raw http request / response to output directory"),
flagSet.DynamicVarP(&options.OutputFile, "output", "o", "proxify_logs.jsonl", "output file to store proxify logs"),
+ flagSet.StringVarP(&options.OutputFormat, "output-format", "of", "jsonl", "output format (jsonl/yaml)"),
flagSet.BoolVar(&options.DumpRequest, "dump-req", false, "Dump only HTTP requests to output file"),
flagSet.BoolVar(&options.DumpResponse, "dump-resp", false, "Dump only HTTP responses to output file"),
- flagSet.BoolVarP(&options.OutputJsonl, "jsonl", "j", false, "write output in JSONL(ines) format"),
flagSet.StringVarP(&options.OutCAFile, "out-ca", "oca", "", "Generate and Save CA File to filename"),
)
@@ -122,7 +124,7 @@ func ParseOptions() (*Options, error) {
silent, verbose, veryVerbose := false, false, false
flagSet.CreateGroup("debug", "debug",
- flagSet.BoolVarP(&options.NoColor, "no-color", "nc", true, "No Color"),
+ flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "No Color"),
flagSet.BoolVar(&options.Version, "version", false, "Version"),
flagSet.BoolVar(&silent, "silent", false, "Silent"),
flagSet.BoolVarP(&verbose, "verbose", "v", false, "Verbose"),
@@ -183,6 +185,11 @@ func ParseOptions() (*Options, error) {
if options.OutputFile == "" {
options.OutputFile = "proxify_logs.jsonl"
}
+
+ if options.OutputFormat == "yaml" {
+ options.OutputFile = strings.ReplaceAll(options.OutputFile, "jsonl", "yaml")
+ }
+
return options, nil
}
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index cbb6ffe0..d7a5a292 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -39,6 +39,7 @@ func NewRunner(options *Options) (*Runner, error) {
ListenAddrHTTP: options.ListenAddrHTTP,
ListenAddrSocks5: options.ListenAddrSocks5,
OutputFile: options.OutputFile,
+ OutputFormat: options.OutputFormat,
OutputDirectory: options.OutputDirectory,
RequestDSL: options.RequestDSL,
ResponseDSL: options.ResponseDSL,
diff --git a/pkg/logger/elastic/elasticsearch.go b/pkg/logger/elastic/elasticsearch.go
index 18a5b305..44c2e02c 100644
--- a/pkg/logger/elastic/elasticsearch.go
+++ b/pkg/logger/elastic/elasticsearch.go
@@ -68,7 +68,7 @@ func New(option *Options) (*Client, error) {
}
// Store saves a passed log event in elasticsearch
-func (c *Client) Save(data types.OutputData) error {
+func (c *Client) Save(data types.HTTPTransaction) error {
var doc map[string]interface{}
if data.Userdata.HasResponse {
doc = map[string]interface{}{
diff --git a/pkg/logger/file/file.go b/pkg/logger/file/file.go
index 622a44ee..2cfd6a91 100644
--- a/pkg/logger/file/file.go
+++ b/pkg/logger/file/file.go
@@ -42,7 +42,7 @@ func New(option *Options) (*Client, error) {
}
// Store writes the log to the file
-func (c *Client) Save(data types.OutputData) error {
+func (c *Client) Save(data types.HTTPTransaction) error {
var err error
logFile := fmt.Sprintf("%s.%s", data.Name, "txt")
if c.options.OutputFolder != "" {
diff --git a/pkg/logger/jsonl/jsonl.go b/pkg/logger/jsonl/jsonl.go
new file mode 100644
index 00000000..785fbf88
--- /dev/null
+++ b/pkg/logger/jsonl/jsonl.go
@@ -0,0 +1,39 @@
+package jsonl
+
+import (
+ "encoding/json"
+ "os"
+ "strings"
+
+ "github.com/projectdiscovery/proxify/pkg/types"
+)
+
+// JsonLinesWriter is a writer for json lines
+type JsonLinesWriter struct {
+ f *os.File
+}
+
+// NewJsonLinesWriter creates a new json lines writer
+func NewJsonLinesWriter(filePath string) (*JsonLinesWriter, error) {
+ file, err := os.Create(filePath)
+ if err != nil {
+ return nil, err
+ }
+ return &JsonLinesWriter{f: file}, nil
+}
+
+// Write writes a http transaction to the file.
+func (j *JsonLinesWriter) Write(data *types.HTTPRequestResponseLog) error {
+ binx, err := json.Marshal(data)
+ if err != nil {
+ return err
+ }
+ _, _ = j.f.WriteString(strings.ReplaceAll(string(binx), "\n", "\\n")) // escape new lines
+ _, _ = j.f.WriteString("\n")
+ return nil
+}
+
+// Close closes the file writer.
+func (j *JsonLinesWriter) Close() error {
+ return j.f.Close()
+}
diff --git a/pkg/logger/kafka/kafka.go b/pkg/logger/kafka/kafka.go
index e6a26dbe..dae01348 100644
--- a/pkg/logger/kafka/kafka.go
+++ b/pkg/logger/kafka/kafka.go
@@ -40,7 +40,7 @@ func New(option *Options) (*Client, error) {
}
// Store passes the message to kafka
-func (c *Client) Save(data types.OutputData) error {
+func (c *Client) Save(data types.HTTPTransaction) error {
msg := &sarama.ProducerMessage{
Topic: c.topic,
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index da5de107..ee28ca7b 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -1,14 +1,11 @@
package logger
import (
- "bytes"
- "encoding/json"
"fmt"
"io"
"net/http"
"net/http/httputil"
"strings"
- "sync"
"time"
"github.com/asaskevich/govalidator"
@@ -16,6 +13,8 @@ import (
"github.com/projectdiscovery/proxify/pkg/logger/elastic"
"github.com/projectdiscovery/proxify/pkg/logger/file"
"github.com/projectdiscovery/proxify/pkg/logger/kafka"
+ "github.com/projectdiscovery/utils/conversion"
+ pdhttpUtils "github.com/projectdiscovery/utils/http"
stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/projectdiscovery/proxify/pkg/types"
@@ -29,35 +28,32 @@ const (
type OptionsLogger struct {
Verbosity types.Verbosity
- OutputFolder string
- OutputFile string
- DumpRequest bool
- DumpResponse bool
- OutputJsonl bool
- MaxSize int
+ OutputFolder string // when output is written to multiple files
+ OutputFile string // when output is written to single file
+ OutputFormat string // jsonl or yaml
+ DumpRequest bool // dump request to file
+ DumpResponse bool // dump response to file
+ MaxSize int // max size of the output
Elastic *elastic.Options
Kafka *kafka.Options
}
type Store interface {
- Save(data types.OutputData) error
+ Save(data types.HTTPTransaction) error
}
type Logger struct {
options *OptionsLogger
- asyncqueue chan types.OutputData
- jsonLogMap sync.Map
+ asyncqueue chan types.HTTPTransaction
Store []Store
+ sWriter OutputFileWriter // sWriter is the structured writer
}
// NewLogger instance
func NewLogger(options *OptionsLogger) *Logger {
logger := &Logger{
options: options,
- asyncqueue: make(chan types.OutputData, 1000),
- }
- if options.OutputJsonl {
- logger.jsonLogMap = sync.Map{}
+ asyncqueue: make(chan types.HTTPTransaction, 1000),
}
if options.Elastic.Addr != "" {
store, err := elastic.New(options.Elastic)
@@ -82,8 +78,6 @@ func NewLogger(options *OptionsLogger) *Logger {
}
store, err := file.New(&file.Options{
OutputFolder: options.OutputFolder,
- OutputJsonl: options.OutputJsonl,
- OutputFile: options.OutputFile,
})
if err != nil {
gologger.Warning().Msgf("Error while creating file logger: %s", err)
@@ -91,73 +85,141 @@ func NewLogger(options *OptionsLogger) *Logger {
logger.Store = append(logger.Store, store)
}
- go logger.AsyncWrite()
+ // setup structured writer
+ if options.OutputFormat != "" {
+ sWriter, err := NewOutputFileWriter(options.OutputFormat, options.OutputFile)
+ if err != nil {
+ gologger.Warning().Msgf("Error while creating structured writer: %s", err)
+ } else {
+ logger.sWriter = sWriter
+ }
+ }
+ go logger.AsyncWrite()
return logger
}
+// LogRequest and user data
+func (l *Logger) LogRequest(req *http.Request, userdata types.UserData) error {
+ // No-op for now , since proxify isn't intended to fail instead return 502
+ // and request can be accessed via response.Request
+ return nil
+}
+
+// LogResponse and user data
+func (l *Logger) LogResponse(resp *http.Response, userdata types.UserData) error {
+ if resp == nil {
+ return nil
+ }
+ // send to writer channel
+ l.asyncqueue <- types.HTTPTransaction{
+ Userdata: userdata,
+ Response: resp,
+ Request: resp.Request,
+ }
+ return nil
+}
+
// AsyncWrite data
func (l *Logger) AsyncWrite() {
- for outputdata := range l.asyncqueue {
- if len(l.Store) > 0 {
- if !l.options.DumpRequest && !l.options.DumpResponse {
- outputdata.PartSuffix = ""
- } else if l.options.DumpRequest && !outputdata.Userdata.HasResponse {
- outputdata.PartSuffix = ".request"
- } else if l.options.DumpResponse && outputdata.Userdata.HasResponse {
- outputdata.PartSuffix = ".response"
+ for httpData := range l.asyncqueue {
+ if httpData.Request == nil {
+ // we can't do anything without request
+ continue
+ }
+ // we have better options to handle this
+ // i.e Buffer reuse and normalizing request/response body (removing encoding etc)
+ reqDump, err := httputil.DumpRequest(httpData.Request, true)
+ if err != nil {
+ gologger.Warning().Msgf("Error while dumping request: %s", err)
+ }
+
+ // debug log request if true
+ l.debugLogRequest(reqDump, httpData.Request)
+
+ var respChain *pdhttpUtils.ResponseChain
+ if httpData.Response != nil {
+ respChainx := pdhttpUtils.NewResponseChain(httpData.Response, 4096)
+ if err := respChainx.Fill(); err == nil {
+ respChain = respChainx
} else {
- continue
- }
- outputdata.Name = fmt.Sprintf("%s%s-%s", outputdata.Userdata.Host, outputdata.PartSuffix, outputdata.Userdata.ID)
- if outputdata.Userdata.HasResponse && !(l.options.DumpRequest || l.options.DumpResponse) {
- if outputdata.Userdata.Match {
- outputdata.Name = outputdata.Name + ".match"
- }
- }
- outputdata.Format = dataWithoutNewLine
- if !strings.HasSuffix(string(outputdata.Data), "\n") {
- outputdata.Format = dataWithNewLine
+ gologger.Warning().Msgf("responseChain: Error while dumping response: %s", err)
}
-
- outputdata.DataString = fmt.Sprintf(outputdata.Format, outputdata.Data)
- if outputdata.Userdata.HasResponse {
- outputdata.Format = "\n" + outputdata.Format
+ }
+ // debug log response if true
+ if respChain != nil {
+ if err := l.debugLogResponse(respChain); err != nil {
+ gologger.Warning().Msgf("Error while logging response: %s", err)
}
- outputdata.RawData = []byte(fmt.Sprintf(outputdata.Format, outputdata.RawData))
+ }
- if l.options.MaxSize > 0 {
- outputdata.DataString = stringsutil.Truncate(outputdata.DataString, l.options.MaxSize)
- outputdata.RawData = []byte(stringsutil.Truncate(string(outputdata.RawData), l.options.MaxSize))
- }
+ // first write to structured writer
+ if l.sWriter != nil {
+ func() {
+ // if matchers were given only store those that match
+ if httpData.Userdata.Match != nil {
+ if !*httpData.Userdata.Match {
+ return
+ }
+ }
- for _, store := range l.Store {
- err := store.Save(outputdata)
+ sData := &types.HTTPRequestResponseLog{
+ Timestamp: time.Now().Format(time.RFC3339),
+ URL: httpData.Request.URL.String(),
+ }
+ defer func() {
+ // write to structured writer with whatever data we have
+ err := l.sWriter.Write(sData)
+ if err != nil {
+ gologger.Warning().Msgf("Error while logging: %s", err)
+ }
+ }()
+ sRequest, err := types.NewHttpRequestData(httpData.Request)
if err != nil {
- gologger.Warning().Msgf("Error while logging: %s", err)
+ gologger.Warning().Msgf("Error while creating request: %s", err)
+ return
+ }
+ sData.Request = sRequest
+ if respChain != nil {
+ sResponse, err := types.NewHttpResponseData(respChain)
+ if err != nil {
+ gologger.Warning().Msgf("Error while creating response: %s", err)
+ }
+ sData.Response = sResponse
}
+ }()
+ }
+
+ // write to other writers
+ if len(l.Store) > 0 {
+ // write request first
+ outputData := httpData
+ // outputData.Data = reqDump
+ outputData.RawData = reqDump
+ outputData.Userdata.HasResponse = false
+ l.storeWriter(outputData)
+
+ // write response if available
+ if respChain != nil {
+ // outputData.Data = respChain.FullResponse().Bytes()
+ outputData.RawData = respChain.FullResponse().Bytes()
+ outputData.Userdata.HasResponse = true
+ l.storeWriter(outputData)
}
}
}
}
-// LogRequest and user data
-func (l *Logger) LogRequest(req *http.Request, userdata types.UserData) error {
- reqdump, err := httputil.DumpRequest(req, true)
- if err != nil {
- return err
- }
- if l.options.OutputJsonl {
- outputData := types.HTTPRequestResponseLog{}
- if err := fillJsonRequestData(req, &outputData); err != nil {
- return err
- }
- l.jsonLogMap.Store(userdata.ID, outputData)
- }
- if l.options.OutputFolder != "" {
- l.asyncqueue <- types.OutputData{RawData: reqdump, Userdata: userdata}
+// Close logger instance
+func (l *Logger) Close() {
+ if l.sWriter != nil {
+ l.sWriter.Close()
}
+ close(l.asyncqueue)
+}
+// debugLogRequest logs the request to the console if debugging is enabled
+func (l *Logger) debugLogRequest(reqdump []byte, req *http.Request) {
if l.options.Verbosity >= types.VerbosityVeryVerbose {
contentType := req.Header.Get("Content-Type")
b, _ := io.ReadAll(req.Body)
@@ -166,121 +228,64 @@ func (l *Logger) LogRequest(req *http.Request, userdata types.UserData) error {
}
gologger.Silent().Msgf("%s", string(reqdump))
}
- return nil
-}
-
-func isASCIICheckRequired(contentType string) bool {
- return stringsutil.ContainsAny(contentType, "application/octet-stream", "application/x-www-form-urlencoded")
}
-// LogResponse and user data
-func (l *Logger) LogResponse(resp *http.Response, userdata types.UserData) error {
- if resp == nil {
- return nil
- }
- respdump, err := httputil.DumpResponse(resp, true)
- if err != nil {
- return err
- }
- respdumpNoBody, err := httputil.DumpResponse(resp, false)
- if err != nil {
- return err
- }
- var data []byte
- if l.options.OutputJsonl {
- defer l.jsonLogMap.Delete(userdata.ID)
- outputData := types.HTTPRequestResponseLog{}
- filledOutputReq, ok := l.jsonLogMap.Load(userdata.ID)
- if !ok {
- if err := fillJsonRequestData(resp.Request, &outputData); err != nil {
- return err
- }
- } else {
- outputData = filledOutputReq.(types.HTTPRequestResponseLog)
- }
- if err := fillJsonResponseData(resp, &outputData); err != nil {
- return err
- }
- data, err = json.Marshal(outputData)
- if err != nil {
- return err
- }
- }
-
- l.asyncqueue <- types.OutputData{RawData: respdump, Data: data, Userdata: userdata}
-
+// debugLogResponse logs the response to the console if debugging is enabled
+func (l *Logger) debugLogResponse(respChain *pdhttpUtils.ResponseChain) error {
if l.options.Verbosity >= types.VerbosityVeryVerbose {
- contentType := resp.Header.Get("Content-Type")
- bodyBytes := bytes.TrimPrefix(respdump, respdumpNoBody)
- if isASCIICheckRequired(contentType) && !govalidator.IsPrintableASCII(string(bodyBytes)) {
- gologger.Silent().Msgf("%s", string(respdumpNoBody))
+ contentType := respChain.Response().Header.Get("Content-Type")
+ if isASCIICheckRequired(contentType) && !govalidator.IsPrintableASCII(conversion.String(respChain.Body().Bytes())) {
+ gologger.Silent().Msgf("%s", respChain.Headers().String())
} else {
- gologger.Silent().Msgf("%s", string(respdump))
+ gologger.Silent().Msgf("%s", respChain.FullResponse().String())
}
}
return nil
}
-// Close logger instance
-func (l *Logger) Close() {
- close(l.asyncqueue)
-}
-
-func fillJsonRequestData(req *http.Request, outputData *types.HTTPRequestResponseLog) error {
- outputData.Timestamp = time.Now().Format(time.RFC3339)
- outputData.URL = req.URL.String()
- // Extract headers from the request
- reqHeaders := make(map[string]string)
- // basic header info
- reqHeaders["scheme"] = req.URL.Scheme
- reqHeaders["method"] = req.Method
- reqHeaders["path"] = req.URL.Path
- reqHeaders["host"] = req.URL.Host
- for key, values := range req.Header {
- reqHeaders[key] = strings.Join(values, ", ")
- }
- outputData.Request.Header = reqHeaders
- // Extract body from the request
- reqBody, err := io.ReadAll(req.Body)
- if err != nil {
- return err
+// storeWriter writes the data to the store (file, kafka, elastic)
+// this can be refactored to make it more readable and scalable
+// with improved interface and probably use of structure http data
+// instead of raw bytes
+func (l *Logger) storeWriter(outputdata types.HTTPTransaction) {
+ if !l.options.DumpRequest && !l.options.DumpResponse {
+ outputdata.PartSuffix = ""
+ } else if l.options.DumpRequest && !outputdata.Userdata.HasResponse {
+ outputdata.PartSuffix = ".request"
+ } else if l.options.DumpResponse && outputdata.Userdata.HasResponse {
+ outputdata.PartSuffix = ".response"
+ } else {
+ return
}
- defer req.Body.Close()
- req.Body = io.NopCloser(strings.NewReader(string(reqBody)))
- if err != nil {
- return err
+ outputdata.Name = fmt.Sprintf("%s%s-%s", outputdata.Userdata.Host, outputdata.PartSuffix, outputdata.Userdata.ID)
+ if outputdata.Userdata.HasResponse && !(l.options.DumpRequest || l.options.DumpResponse) {
+ if outputdata.Userdata.Match != nil && *outputdata.Userdata.Match {
+ outputdata.Name = outputdata.Name + ".match"
+ }
}
- outputData.Request.Body = string(reqBody)
- // Extract raw request
- reqdumpNoBody, err := httputil.DumpRequest(req, false)
- if err != nil {
- return err
+ outputdata.Format = dataWithoutNewLine
+ if !strings.HasSuffix(string(outputdata.Data), "\n") {
+ outputdata.Format = dataWithNewLine
}
- outputData.Request.Raw = string(reqdumpNoBody)
- return nil
-}
-func fillJsonResponseData(resp *http.Response, outputData *types.HTTPRequestResponseLog) error {
- outputData.Timestamp = time.Now().Format(time.RFC3339)
- // Extract headers from the response
- respHeaders := make(map[string]string)
- for key, values := range resp.Header {
- respHeaders[key] = strings.Join(values, ", ")
+ outputdata.DataString = fmt.Sprintf(outputdata.Format, outputdata.Data)
+ if outputdata.Userdata.HasResponse {
+ outputdata.Format = "\n" + outputdata.Format
}
- outputData.Response.Header = respHeaders
- // Extract body from the response
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return err
+ outputdata.RawData = []byte(fmt.Sprintf(outputdata.Format, outputdata.RawData))
+
+ if l.options.MaxSize > 0 {
+ outputdata.DataString = stringsutil.Truncate(outputdata.DataString, l.options.MaxSize)
+ outputdata.RawData = []byte(stringsutil.Truncate(string(outputdata.RawData), l.options.MaxSize))
}
- defer resp.Body.Close()
- resp.Body = io.NopCloser(strings.NewReader(string(respBody)))
- outputData.Response.Body = string(respBody)
- // Extract raw response
- respdumpNoBody, err := httputil.DumpResponse(resp, false)
- if err != nil {
- return err
+ for _, store := range l.Store {
+ err := store.Save(outputdata)
+ if err != nil {
+ gologger.Warning().Msgf("Error while logging: %s", err)
+ }
}
- outputData.Response.Raw = string(respdumpNoBody)
- return nil
+}
+
+func isASCIICheckRequired(contentType string) bool {
+ return stringsutil.ContainsAny(contentType, "application/octet-stream", "application/x-www-form-urlencoded")
}
diff --git a/pkg/logger/writer.go b/pkg/logger/writer.go
new file mode 100644
index 00000000..61372af4
--- /dev/null
+++ b/pkg/logger/writer.go
@@ -0,0 +1,38 @@
+package logger
+
+import (
+ "errors"
+
+ "github.com/projectdiscovery/proxify/pkg/logger/jsonl"
+ "github.com/projectdiscovery/proxify/pkg/logger/yaml"
+ "github.com/projectdiscovery/proxify/pkg/types"
+)
+
+var (
+ _ OutputFileWriter = &jsonl.JsonLinesWriter{}
+ // multi doc yaml writer with --- separator
+ _ OutputFileWriter = &yaml.YamlMultiDocWriter{}
+
+ ErrorInvalidFormat = errors.New("invalid format: expected jsonl or yaml")
+)
+
+// OutputFileWriter is an interface for writing structured
+// data to a file.
+type OutputFileWriter interface {
+ // Write writes a http transaction to the file.
+ Write(data *types.HTTPRequestResponseLog) error
+ // Close closes the file writer.
+ Close() error
+}
+
+// NewOutputFileWriter creates a new output file writer
+func NewOutputFileWriter(format, filePath string) (OutputFileWriter, error) {
+ switch format {
+ case "jsonl":
+ return jsonl.NewJsonLinesWriter(filePath)
+ case "yaml":
+ return yaml.NewYamlMultiDocWriter(filePath)
+ default:
+ return nil, ErrorInvalidFormat
+ }
+}
diff --git a/pkg/logger/yaml/yaml.go b/pkg/logger/yaml/yaml.go
new file mode 100644
index 00000000..d83464b5
--- /dev/null
+++ b/pkg/logger/yaml/yaml.go
@@ -0,0 +1,43 @@
+package yaml
+
+import (
+ "os"
+
+ "github.com/goccy/go-yaml"
+ "github.com/projectdiscovery/proxify/pkg/types"
+)
+
+// YamlMultiDocWriter is a writer for yaml multi doc
+type YamlMultiDocWriter struct {
+ f *os.File
+ enc *yaml.Encoder
+}
+
+// NewYamlMultiDocWriter creates a new yaml multi doc writer
+func NewYamlMultiDocWriter(filePath string) (*YamlMultiDocWriter, error) {
+ file, err := os.Create(filePath)
+ if err != nil {
+ return nil, err
+ }
+ enc := yaml.NewEncoder(file, yaml.UseLiteralStyleIfMultiline(true), yaml.UseSingleQuote(true))
+ return &YamlMultiDocWriter{f: file, enc: enc}, nil
+}
+
+// Write writes a http transaction to the file.
+func (y *YamlMultiDocWriter) Write(data *types.HTTPRequestResponseLog) error {
+ if err := y.enc.Encode(data); err != nil {
+ return err
+ }
+ return nil
+}
+
+// Close closes the file writer.
+func (y *YamlMultiDocWriter) Close() error {
+ if y.enc != nil {
+ y.enc.Close()
+ }
+ if y.f != nil {
+ _ = y.f.Close()
+ }
+ return nil
+}
diff --git a/pkg/types/userdata.go b/pkg/types/userdata.go
index 1edd6d43..44d44b65 100644
--- a/pkg/types/userdata.go
+++ b/pkg/types/userdata.go
@@ -1,13 +1,27 @@
package types
+import (
+ "io"
+ "net/http"
+ "net/http/httputil"
+ "strings"
+
+ pdhttpUtils "github.com/projectdiscovery/utils/http"
+)
+
+// UserData is context used to identify a http
+// transaction and its state like match, response etc.
type UserData struct {
ID string
- Match bool
+ Match *bool
HasResponse bool
Host string
}
-type OutputData struct {
+// HTTPTransaction is a struct for http transaction
+// it contains data of every request/response obtained
+// from proxy
+type HTTPTransaction struct {
Userdata UserData
RawData []byte
Data []byte
@@ -15,19 +29,80 @@ type OutputData struct {
Name string
PartSuffix string
Format string
+
+ Request *http.Request
+ Response *http.Response
}
+// HTTPRequestResponseLog is a struct for http request and response log
+// it is a processed version of http transaction for logging
+// in more structured format than just raw bytes
type HTTPRequestResponseLog struct {
- Timestamp string `json:"timestamp,omitempty"`
- URL string `json:"url,omitempty"`
- Request struct {
- Header map[string]string `json:"header,omitempty"`
- Body string `json:"body,omitempty"`
- Raw string `json:"raw,omitempty"`
- } `json:"request,omitempty"`
- Response struct {
- Header map[string]string `json:"header,omitempty"`
- Body string `json:"body,omitempty"`
- Raw string `json:"raw,omitempty"`
- } `json:"response,omitempty"`
+ Timestamp string `json:"timestamp,omitempty"`
+ URL string `json:"url,omitempty"`
+ Request *HTTPRequest `json:"request,omitempty"`
+ Response *HTTPResponse `json:"response,omitempty"`
+}
+
+// HTTPRequest is a struct for http request
+type HTTPRequest struct {
+ Header map[string]string `json:"header,omitempty"`
+ Body string `json:"body,omitempty"`
+ Raw string `json:"raw,omitempty"`
+}
+
+// NewHttpRequestData creates a new HttpRequest with data extracted from an http.Request
+func NewHttpRequestData(req *http.Request) (*HTTPRequest, error) {
+ httpRequest := &HTTPRequest{
+ Header: make(map[string]string),
+ }
+
+ // Extract headers from the request
+ httpRequest.Header["scheme"] = req.URL.Scheme
+ httpRequest.Header["method"] = req.Method
+ httpRequest.Header["path"] = req.URL.Path
+ httpRequest.Header["host"] = req.URL.Host
+ for key, values := range req.Header {
+ httpRequest.Header[key] = strings.Join(values, ", ")
+ }
+
+ // Extract body from the request
+ reqBody, err := io.ReadAll(req.Body)
+ if err != nil {
+ return nil, err
+ }
+ defer req.Body.Close()
+ req.Body = io.NopCloser(strings.NewReader(string(reqBody)))
+ httpRequest.Body = string(reqBody)
+
+ // Extract raw request
+ reqdumpNoBody, err := httputil.DumpRequest(req, false)
+ if err != nil {
+ return nil, err
+ }
+ httpRequest.Raw = string(reqdumpNoBody)
+
+ return httpRequest, nil
+}
+
+// HTTPResponse is a struct for http response
+type HTTPResponse struct {
+ Header map[string]string `json:"header,omitempty"`
+ Body string `json:"body,omitempty"`
+ Raw string `json:"raw,omitempty"`
+}
+
+// NewHttpResponseData creates a new HttpResponse with data extracted from an http.Response
+func NewHttpResponseData(resp *pdhttpUtils.ResponseChain) (*HTTPResponse, error) {
+ httpResponse := &HTTPResponse{
+ Header: make(map[string]string),
+ }
+ // Extract headers from the response
+ for key, values := range resp.Response().Header {
+ httpResponse.Header[key] = strings.Join(values, ", ")
+ }
+ httpResponse.Body = resp.Body().String()
+ httpResponse.Raw = resp.Headers().String() // doesn't include body
+
+ return httpResponse, nil
}
diff --git a/pkg/util/util.go b/pkg/util/util.go
index d8bc7db3..44221d94 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -10,8 +10,8 @@ import (
"strings"
)
-// HTTPRequesToMap Converts HTTP Request to Matcher Map
-func HTTPRequesToMap(req *http.Request) (map[string]interface{}, error) {
+// HTTPRequestToMap Converts HTTP Request to Matcher Map
+func HTTPRequestToMap(req *http.Request) (map[string]interface{}, error) {
m := make(map[string]interface{})
var headers string
for k, v := range req.Header {
@@ -37,6 +37,12 @@ func HTTPRequesToMap(req *http.Request) (map[string]interface{}, error) {
reqdumpString := string(reqdump)
m["raw"] = reqdumpString
m["request"] = reqdumpString
+ m["method"] = req.Method
+ m["path"] = req.URL.Path
+ m["host"] = req.URL.Host
+ m["scheme"] = req.URL.Scheme
+ m["url"] = req.URL.String()
+ m["query"] = req.URL.Query().Encode()
return m, nil
}
@@ -82,3 +88,15 @@ func MatchAnyRegex(regexes []string, data string) bool {
}
return false
}
+
+// EvalBoolSlice evaluates a slice of bools using a logical AND
+func EvalBoolSlice(slice []bool) bool {
+ if len(slice) == 0 {
+ return false
+ }
+ result := slice[0]
+ for _, b := range slice {
+ result = result && b
+ }
+ return result
+}
diff --git a/proxy.go b/proxy.go
index dfa15560..e3e84bd0 100644
--- a/proxy.go
+++ b/proxy.go
@@ -9,6 +9,7 @@ import (
"log"
"net"
"net/http"
+ "net/http/httptest"
"net/http/httputil"
"net/url"
"os"
@@ -33,6 +34,8 @@ import (
rbtransport "github.com/projectdiscovery/roundrobin/transport"
"github.com/projectdiscovery/tinydns"
errorutil "github.com/projectdiscovery/utils/errors"
+ readerUtil "github.com/projectdiscovery/utils/reader"
+ stringsutil "github.com/projectdiscovery/utils/strings"
"golang.org/x/net/proxy"
)
@@ -51,6 +54,7 @@ type Options struct {
ListenAddrSocks5 string
OutputDirectory string
OutputFile string
+ OutputFormat string
RequestDSL []string
ResponseDSL []string
UpstreamHTTPProxies []string
@@ -81,6 +85,116 @@ type Proxy struct {
tinydns *tinydns.TinyDNS
rbhttp *rbtransport.RoundTransport
rbsocks5 *rbtransport.RoundTransport
+ proxifyMux *http.ServeMux // serve banner page and static files
+ listenAddr string
+}
+
+func NewProxy(options *Options) (*Proxy, error) {
+
+ switch options.Verbosity {
+ case types.VerbositySilent:
+ martianlog.SetLevel(martianlog.Silent)
+ case types.VerbosityVerbose:
+ martianlog.SetLevel(martianlog.Info)
+ case types.VerbosityVeryVerbose:
+ martianlog.SetLevel(martianlog.Debug)
+ default:
+ martianlog.SetLevel(martianlog.Error)
+ }
+
+ logger := logger.NewLogger(&logger.OptionsLogger{
+ Verbosity: options.Verbosity,
+ OutputFile: options.OutputFile,
+ OutputFormat: options.OutputFormat,
+ OutputFolder: options.OutputDirectory,
+ DumpRequest: options.DumpRequest,
+ DumpResponse: options.DumpResponse,
+ MaxSize: options.MaxSize,
+ Elastic: options.Elastic,
+ Kafka: options.Kafka,
+ })
+
+ var tdns *tinydns.TinyDNS
+
+ fastdialerOptions := fastdialer.DefaultOptions
+ fastdialerOptions.EnableFallback = true
+ fastdialerOptions.Deny = options.Deny
+ fastdialerOptions.Allow = options.Allow
+ if options.ListenDNSAddr != "" {
+ dnsmapping := make(map[string]*tinydns.DnsRecord)
+ for _, record := range strings.Split(options.DNSMapping, ",") {
+ data := strings.Split(record, ":")
+ if len(data) != 2 {
+ continue
+ }
+ dnsmapping[data[0]] = &tinydns.DnsRecord{A: []string{data[1]}}
+ }
+ var err error
+ tdns, err = tinydns.New(&tinydns.Options{
+ ListenAddress: options.ListenDNSAddr,
+ Net: "udp",
+ UpstreamServers: []string{options.DNSFallbackResolver},
+ DnsRecords: dnsmapping,
+ })
+ if err != nil {
+ return nil, err
+ }
+ fastdialerOptions.BaseResolvers = []string{"127.0.0.1" + options.ListenDNSAddr}
+ }
+ dialer, err := fastdialer.NewDialer(fastdialerOptions)
+ if err != nil {
+ return nil, err
+ }
+
+ var rbhttp, rbsocks5 *rbtransport.RoundTransport
+ if len(options.UpstreamHTTPProxies) > 0 {
+ rbhttp, err = rbtransport.NewWithOptions(options.UpstreamProxyRequestsNumber, options.UpstreamHTTPProxies...)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if len(options.UpstreamSock5Proxies) > 0 {
+ rbsocks5, err = rbtransport.NewWithOptions(options.UpstreamProxyRequestsNumber, options.UpstreamSock5Proxies...)
+ if err != nil {
+ return nil, err
+ }
+ }
+ pmux, err := getProxifyServerMux()
+ if err != nil {
+ return nil, err
+ }
+
+ proxy := &Proxy{
+ logger: logger,
+ options: options,
+ Dialer: dialer,
+ tinydns: tdns,
+ rbhttp: rbhttp,
+ rbsocks5: rbsocks5,
+ proxifyMux: pmux,
+ }
+
+ if err := proxy.setupHTTPProxy(); err != nil {
+ return nil, err
+ }
+
+ var socks5proxy *socks5.Server
+ if options.ListenAddrSocks5 != "" {
+ socks5Config := &socks5.Config{
+ Dial: proxy.httpTunnelDialer,
+ }
+ if options.Verbosity <= types.VerbositySilent {
+ socks5Config.Logger = log.New(io.Discard, "", log.Ltime|log.Lshortfile)
+ }
+ socks5proxy, err = socks5.New(socks5Config)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ proxy.socks5proxy = socks5proxy
+
+ return proxy, nil
}
// ModifyRequest
@@ -94,21 +208,31 @@ func (p *Proxy) ModifyRequest(req *http.Request) error {
Host: req.Host,
}
+ if stringsutil.EqualFoldAny(req.Host, "proxify", "proxify:443", "proxify:80", p.listenAddr) {
+ // hijack if this is true
+ return p.hijackNServe(req, ctx)
+ }
+
// If callbacks are given use them (for library use cases)
if p.options.OnRequestCallback != nil {
return p.options.OnRequestCallback(req, ctx)
}
+ boolSlice := []bool{}
for _, expr := range p.options.RequestDSL {
- if !userData.Match {
- m, _ := util.HTTPRequesToMap(req)
- v, err := dsl.EvalExpr(expr, m)
- if err != nil {
- gologger.Warning().Msgf("Could not evaluate request dsl: %s\n", err)
- }
- userData.Match = err == nil && v.(bool)
+ m, _ := util.HTTPRequestToMap(req)
+ v, err := dsl.EvalExpr(expr, m)
+ if err != nil {
+ gologger.Warning().Msgf("Could not evaluate request dsl: %s\n", err)
}
+ boolSlice = append(boolSlice, err == nil && v.(bool))
+ }
+ // evaluate bool array to get match status
+ if len(boolSlice) > 0 {
+ tmp := util.EvalBoolSlice(boolSlice)
+ userData.Match = &tmp
}
+
ctx.Set("user-data", userData)
// perform match and replace
@@ -129,31 +253,39 @@ func (p *Proxy) ModifyResponse(resp *http.Response) error {
}
}
if userData == nil {
- gologger.Error().Msgf("something went wrong got response without userData")
+ gologger.Warning().Msgf("something went wrong got response without userData")
// pass empty struct to avoid panic
userData = &types.UserData{}
}
userData.HasResponse = true
+ // if content-length is zero and remove header
+ if resp.ContentLength == 0 {
+ resp.Header.Del("Content-Length")
+ }
+
// If callbacks are given use them (for library use cases)
if p.options.OnResponseCallback != nil {
return p.options.OnResponseCallback(resp, ctx)
}
- // TODO: match in request seems to be seperate from response
- // but share same `Match` value. investigate this
- matchStatus := false
+ boolSlice := []bool{}
for _, expr := range p.options.ResponseDSL {
- if !matchStatus {
- m, _ := util.HTTPResponseToMap(resp)
- v, err := dsl.EvalExpr(expr, m)
- if err != nil {
- gologger.Warning().Msgf("Could not evaluate response dsl: %s\n", err)
- }
- matchStatus = err == nil && v.(bool)
+ m, _ := util.HTTPResponseToMap(resp)
+ v, err := dsl.EvalExpr(expr, m)
+ if err != nil {
+ gologger.Warning().Msgf("Could not evaluate response dsl: %s\n", err)
+ }
+ boolSlice = append(boolSlice, err == nil && v.(bool))
+ }
+ if len(boolSlice) > 0 {
+ tmp := util.EvalBoolSlice(boolSlice)
+ // finalize
+ if userData.Match != nil {
+ tmp = *userData.Match && tmp
}
+ userData.Match = &tmp
}
- userData.Match = matchStatus
// perform match and replace
if len(p.options.ResponseMatchReplaceDSL) != 0 {
_ = p.MatchReplaceResponse(resp)
@@ -212,8 +344,8 @@ func (p *Proxy) MatchReplaceRequest(req *http.Request) error {
// MatchReplaceRequest strings or regex
func (p *Proxy) MatchReplaceResponse(resp *http.Response) error {
- // Set Content-Length to zero to allow automatic calculation
- resp.ContentLength = 0
+ // // Set Content-Length to zero to allow automatic calculation
+ resp.ContentLength = -1
// lazy mode - dump request
respdump, err := httputil.DumpResponse(resp, true)
@@ -244,8 +376,14 @@ func (p *Proxy) MatchReplaceResponse(resp *http.Response) error {
// closes old body to allow memory reuse
resp.Body.Close()
resp.Header = responseNew.Header
- resp.Body = responseNew.Body
- resp.ContentLength = responseNew.ContentLength
+ resp.Body, err = readerUtil.NewReusableReadCloser(responseNew.Body)
+ if err != nil {
+ return err
+ }
+ if resp.ContentLength == 0 {
+ resp.Header.Del("Content-Length")
+ }
+ // resp.ContentLength = responseNew.ContentLength
return nil
}
@@ -276,18 +414,10 @@ func (p *Proxy) Run() error {
if err != nil {
gologger.Fatal().Msgf("failed to setup listener got %v", err)
}
- // serve web page to download ca cert
+ p.listenAddr = l.Addr().String()
wg.Add(1)
go func() {
defer wg.Done()
-
- gologger.Fatal().Msgf("%v", serveWebPage(l))
- }()
-
- wg.Add(1)
- go func() {
- defer wg.Done()
-
gologger.Fatal().Msgf("%v", p.httpProxy.Serve(l))
}()
}
@@ -322,6 +452,8 @@ func (p *Proxy) Run() error {
return nil
}
+func (p *Proxy) Stop() {}
+
// setupHTTPProxy configures proxy with settings
func (p *Proxy) setupHTTPProxy() error {
hp := martian.NewProxy()
@@ -379,121 +511,31 @@ func (p *Proxy) getRoundTripper() (http.RoundTripper, error) {
return roundtrip, nil
}
-func (p *Proxy) Stop() {
- // p.httpProxy.Close()
+func (p *Proxy) httpTunnelDialer(ctx context.Context, network, addr string) (net.Conn, error) {
+ return p.socks5tunnel.MakeTunnel(nil, nil, p.bufioPool, addr)
}
-func NewProxy(options *Options) (*Proxy, error) {
-
- switch options.Verbosity {
- case types.VerbositySilent:
- martianlog.SetLevel(martianlog.Silent)
- case types.VerbosityVerbose:
- martianlog.SetLevel(martianlog.Info)
- case types.VerbosityVeryVerbose:
- martianlog.SetLevel(martianlog.Debug)
- default:
- martianlog.SetLevel(martianlog.Error)
- }
-
- logger := logger.NewLogger(&logger.OptionsLogger{
- Verbosity: options.Verbosity,
- OutputFile: options.OutputFile,
- OutputFolder: options.OutputDirectory,
- DumpRequest: options.DumpRequest,
- DumpResponse: options.DumpResponse,
- OutputJsonl: options.OutputJsonl,
- MaxSize: options.MaxSize,
- Elastic: options.Elastic,
- Kafka: options.Kafka,
- })
-
- var tdns *tinydns.TinyDNS
-
- fastdialerOptions := fastdialer.DefaultOptions
- fastdialerOptions.EnableFallback = true
- fastdialerOptions.Deny = options.Deny
- fastdialerOptions.Allow = options.Allow
- if options.ListenDNSAddr != "" {
- dnsmapping := make(map[string]*tinydns.DnsRecord)
- for _, record := range strings.Split(options.DNSMapping, ",") {
- data := strings.Split(record, ":")
- if len(data) != 2 {
- continue
- }
- dnsmapping[data[0]] = &tinydns.DnsRecord{A: []string{data[1]}}
- }
- var err error
- tdns, err = tinydns.New(&tinydns.Options{
- ListenAddress: options.ListenDNSAddr,
- Net: "udp",
- UpstreamServers: []string{options.DNSFallbackResolver},
- DnsRecords: dnsmapping,
- })
- if err != nil {
- return nil, err
- }
- fastdialerOptions.BaseResolvers = []string{"127.0.0.1" + options.ListenDNSAddr}
- }
- dialer, err := fastdialer.NewDialer(fastdialerOptions)
+func (p *Proxy) hijackNServe(req *http.Request, ctx *martian.Context) error {
+ conn, brw, err := ctx.Session().Hijack()
if err != nil {
- return nil, err
- }
-
- var rbhttp, rbsocks5 *rbtransport.RoundTransport
- if len(options.UpstreamHTTPProxies) > 0 {
- rbhttp, err = rbtransport.NewWithOptions(options.UpstreamProxyRequestsNumber, options.UpstreamHTTPProxies...)
- if err != nil {
- return nil, err
- }
- }
- if len(options.UpstreamSock5Proxies) > 0 {
- rbsocks5, err = rbtransport.NewWithOptions(options.UpstreamProxyRequestsNumber, options.UpstreamSock5Proxies...)
- if err != nil {
- return nil, err
- }
- }
-
- proxy := &Proxy{
- logger: logger,
- options: options,
- Dialer: dialer,
- tinydns: tdns,
- rbhttp: rbhttp,
- rbsocks5: rbsocks5,
- }
-
- if err := proxy.setupHTTPProxy(); err != nil {
- return nil, err
+ return err
}
-
- var socks5proxy *socks5.Server
- if options.ListenAddrSocks5 != "" {
- socks5Config := &socks5.Config{
- Dial: proxy.httpTunnelDialer,
- }
- if options.Verbosity <= types.VerbositySilent {
- socks5Config.Logger = log.New(io.Discard, "", log.Ltime|log.Lshortfile)
- }
- socks5proxy, err = socks5.New(socks5Config)
- if err != nil {
- return nil, err
- }
+ defer conn.Close()
+ rec := httptest.NewRecorder()
+ p.proxifyMux.ServeHTTP(rec, req)
+ resp := rec.Result()
+ resp.Close = true
+ if err := resp.Write(brw); err != nil {
+ gologger.Warning().Msgf("failed to write response: %v", err)
}
-
- proxy.socks5proxy = socks5proxy
-
- return proxy, nil
-}
-
-func (p *Proxy) httpTunnelDialer(ctx context.Context, network, addr string) (net.Conn, error) {
- return p.socks5tunnel.MakeTunnel(nil, nil, p.bufioPool, addr)
+ brw.Flush()
+ return nil
}
-func serveWebPage(l net.Listener) error {
+func getProxifyServerMux() (*http.ServeMux, error) {
cwd, err := os.Getwd()
if err != nil {
- return fmt.Errorf("failed to get current working directory: %v", err)
+ return nil, fmt.Errorf("failed to get current working directory: %v", err)
}
absStaticDirPath := strings.Join([]string{strings.Split(cwd, "cmd")[0], "static"}, "/")
@@ -505,20 +547,16 @@ func serveWebPage(l net.Listener) error {
buffer, err := certs.GetRawCA()
if err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
- gologger.Error().Msgf("failed to get raw CA: %v", err)
+ gologger.Warning().Msgf("failed to get raw CA: %v", err)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", "attachment; filename=\"proxify.pem\"")
if _, err := w.Write(buffer.Bytes()); err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
- gologger.Error().Msgf("failed to write raw CA: %v", err)
+ gologger.Warning().Msgf("failed to write raw CA: %v", err)
return
}
})
-
- server := &http.Server{
- Handler: mux,
- }
- return server.Serve(l)
+ return mux, nil
}