From de607531fe91cd5c813f80dde3081a1d739e0b34 Mon Sep 17 00:00:00 2001 From: Dmytro Biloshytskyi Date: Tue, 13 Aug 2024 13:15:19 +0300 Subject: [PATCH] Added README.md. Code cleanup --- .gitignore | 3 +- README.md | 111 ++++++++++++++++++++++++++++++++++++ entrypoint.sh | 2 +- main.go | 2 +- nginx/default.conf.template | 2 +- nginx/nginx.conf | 17 ++---- 6 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 README.md diff --git a/.gitignore b/.gitignore index 418a7b8..824d287 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ go.work.sum .DS_Store .lh/ -.vscode/ \ No newline at end of file +.vscode/ +jrpc-interceptor \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..371bffd --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +# Json RPC interceptor + +The tool allows to intercept HTTP (Json RPC) requests and publish the results as Prometheus metrics. + +## How it works +1. The interceptor is Nginx proxy server that listens on a specific port and forwards the requests to the JRPC app. +2. Nginx contains Lua script that parses the requests to find out JRPC methods. +3. Nginx sends the logs and other metrics (e.g. response time and parsed JRPC method) to syslog server in specific format. +4. Syslog server is Go application that parses that logs and publishes the metrics to Prometheus. + +Your JRPC app and the interceptor should be running on the same machine (network). + + +## How to run +1. Build the interceptor docker image: +```bash +docker build -t jrpc-interceptor . +``` +2. Run your JRPC app, e.g.: +```bash +docker run -p 8545:8545 -it horax/erigon:latest bash +``` +3. Run the interceptor: +On MacOS: +```bash +docker run -p 514:514/udp -p 9100:9100 -e LISTEN_PORT=80 -e SERVICE_TO_PROXY=192.168.1.2:8545 -e LOG_SERVER_URL=0.0.0.0:514 -p 8081:80 jrpc-interceptor +``` +Where `192.168.1.2` is the IP of the JRPC app. + +On Linux: +```bash +docker run --net="host" -e LISTEN_PORT=0.0.0.0:8081 -e SERVICE_TO_PROXY=0.0.0.0:8545 -e LOG_SERVER_URL=0.0.0.0:514 jrpc-interceptor +``` + +Where: +- `LISTEN_PORT` - the port / IP where the interceptor listens for incoming requests. +- `SERVICE_TO_PROXY` - the IP / port of the JRPC app. +- `LOG_SERVER_URL` - the IP / port of the syslog server. +- `PROMETHEUS_URL` - the IP / port of the Prometheus server. Optional, "0.0.0.0:9100" by default. +- `USE_PROMETHEUS` - whether to publish metrics to Prometheus. Optional, "true" by default. +- `LOG_SERVER_DEBUG` - whether to print the logs to stdout. Optional, "true" by default. + +4. Send requests to the interceptor: +```bash +curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://localhost:8081 +``` +5. Check the logs. You should see something like this: +```bash +2024/08/13 09:01:37 ef01982f3841: 192.168.65.1|http|localhost|POST|HTTP/1.1|/|200|0.002|193|326|0.003|eth_blockNumber +``` +6. Open URL `http://localhost:9100/metrics` in your browser to see the metrics. + +## Local development +1. Add download dependencies: +```bash +go mod download +``` +2. Build the package +```bash +go build . +``` +3. Run the interceptor: +```bash +./jrpc-interceptor -debug=${LOG_SERVER_DEBUG:-true} -listenSyslog=${LOG_SERVER_URL:-"0.0.0.0:514"} -listenHTTP=${PROMETHEUS_URL:-"0.0.0.0:9100"} -usePrometheus=${USE_PROMETHEUS:-true} +``` + +## Nginx configuration +All the configuration can be found inside the `nginx` folder. + +Basic info: + +- `worker_processes` sets to 4; +- `worker_connections` sets 2048; +- `client_body_buffer_size` sets to `10M`;; +- `client_max_body_size` sets to `1000M` due to the large size of some jrpc requests; + +Used `openresty/openresty:alpine` to be able to use Lua scripts. + +### Syslog format +The interceptor sends the logs to the syslog server in the following format: +``` +$remote_addr|$scheme|$host|$request_method|$server_protocol|$request_uri|$status|$request_time|$request_length|$bytes_sent|$upstream_response_time|$jrpc_method +``` +That returns something like +``` +2024/08/13 09:01:37 ef01982f3841: 192.168.65.1|http|localhost|POST|HTTP/1.1|/|200|0.002|193|326|0.003|eth_blockNumber +``` + +### Limitations and workarounds +- All the original headers passes as is, except `Host` header. It's replaced with the `SERVICE_TO_PROXY` value. + +## Errors and Troubleshooting +Instead of a valid `jrpc_method` you can show following errors: +- `no_method_field` this means that original request contains `jsonrpc` but doesn't contain json rpc `method` field; +- `invalid_jsonrpc_request` the request contains `jsonrpc` field but it's not a valid json rpc request (cannot decode json body); +- `no_jsonrpc_field` the request does not contain `jsonrpc` field; + +To see the errors you can use `LOG_SERVER_DEBUG=true` environment variable. To check full json logs, log in to container and check `/var/log/nginx/access_with_body.json` file. + +## Metrics +The interceptor publishes the following metrics: +``` +- `ngx_request_count` - the number of requests; +- `ngx_request_size_bytes` - the size of the request; +- `ngx_response_size_bytes` - the size of the response; +- `ngx_request_duration_seconds` - the time of the request; + +For the `ngx_request_duration_seconds` metric, we use `$request_time` value. +It's the time between the first bytes were read from the client and the log write after the last bytes were sent to the client. + + diff --git a/entrypoint.sh b/entrypoint.sh index ef14476..179ea43 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -./jrpc-interceptor -debug true & +./jrpc-interceptor -debug=${LOG_SERVER_DEBUG:-true} -listenSyslog=${LOG_SERVER_URL:-"0.0.0.0:514"} -listenHTTP=${PROMETHEUS_URL:-"0.0.0.0:9100"} -usePrometheus=${USE_PROMETHEUS:-true} & envsubst '${SERVICE_TO_PROXY} ${LOG_SERVER_URL} ${LISTEN_PORT}' < /etc/nginx/templates/nginx.conf.template > /etc/nginx/conf.d/default.conf && exec nginx -g 'daemon off;' \ No newline at end of file diff --git a/main.go b/main.go index 276a251..bf9181e 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( var ( debug bool - usePrometheus bool ) func receiveSyslog(ch syslog.LogPartsChannel) { @@ -43,6 +42,7 @@ func main() { var ( listenSyslog string listenHTTP string + usePrometheus bool ) flag.StringVar(&listenSyslog, "listenSyslog", "0.0.0.0:514", "ip:port to listen for syslog messages") diff --git a/nginx/default.conf.template b/nginx/default.conf.template index e910767..32a3500 100644 --- a/nginx/default.conf.template +++ b/nginx/default.conf.template @@ -6,7 +6,7 @@ upstream eth-client { server { listen ${LISTEN_PORT}; # Declare the custom variables - set $custom_request_body ""; + set $jrpc_method ""; set $custom_request_headers ""; access_log /var/log/nginx/access_with_body.json json_combined; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index eac425c..009da14 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -9,12 +9,7 @@ http { client_max_body_size 1000M; client_body_buffer_size 10M; - log_format timed '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for" ' - 'rt=$request_time'; - - log_format collector '$remote_addr|$scheme|$host|$request_method|$server_protocol|$request_uri|$status|$request_time|$request_length|$bytes_sent|$upstream_response_time|$custom_request_body'; + log_format collector '$remote_addr|$scheme|$host|$request_method|$server_protocol|$request_uri|$status|$request_time|$request_length|$bytes_sent|$upstream_response_time|$jrpc_method'; log_format json_combined escape=json '{' '"remote_addr": "$remote_addr", ' @@ -29,7 +24,7 @@ http { '"request_time": "$request_time", ' '"response_time": "$upstream_response_time", ' '"headers": "$custom_request_headers", ' - '"jrpc_method": "$custom_request_body"' + '"jrpc_method": "$jrpc_method"' '},'; include /etc/nginx/conf.d/*.conf; @@ -42,18 +37,18 @@ http { local ok, json_data = pcall(cjson.decode, data) if ok then if json_data.method then - ngx.var.custom_request_body = json_data.method + ngx.var.jrpc_method = json_data.method else if data then ngx.log(ngx.ERR, "Request: ", data) end - ngx.var.custom_request_body = "no_method_field" + ngx.var.jrpc_method = "no_method_field" end else - ngx.var.custom_request_body = "invalid_jsonrpc_request: " .. tostring(json_data) + ngx.var.jrpc_method = "invalid_jsonrpc_request: " .. tostring(json_data) end else - ngx.var.custom_request_body = "no_jsonrpc_field" + ngx.var.jrpc_method = "no_jsonrpc_field" end -- Capture request headers