feat: add script to generate CU calc record rules

monitoring/rules.cu.geth.yml

@@ -0,0 +1,171 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: subway-cu-record-geth
+  namespace: proxies
+spec:
+  groups:
+  - name: subway-cu-record-geth.rules
+    rules:
+    - expr: subway_rpc_calls_finished{method="eth_chainId"} * 2
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_syncing"} * 2
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_coinbase"} * 2
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_accounts"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_blockNumber"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_gasPrice"} * 19
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_blobBaseFee"} * 19
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_maxPriorityFeePerGas"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_feeHistory"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_sendRawTransaction"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_call"} * 26
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_estimateGas"} * 87
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_createAccessList"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBlockByHash"} * 21
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBlockByNumber"} * 16
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBlockTransactionCountByHash"}
+        * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBlockTransactionCountByNumber"}
+        * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getUncleCountByBlockHash"} * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getUncleCountByBlockNumber"} * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBlockReceipts"} * 500
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_newFilter"} * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_newBlockFilter"} * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_newPendingTransactionFilter"} *
+        20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getFilterChanges"} * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getFilterLogs"} * 75
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getLogs"} * 75
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_uninstallFilter"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getBalance"} * 19
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getStorageAt"} * 17
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getTransactionCount"} * 26
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getCode"} * 19
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getProof"} * 21
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getTransactionByHash"} * 17
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getTransactionByBlockHashAndIndex"}
+        * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getTransactionByBlockNumberAndIndex"}
+        * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="eth_getTransactionReceipt"} * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getBadBlocks"} * 10
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getRawBlock"} * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getRawHeader"} * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getRawReceipts"} * 500
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getRawTransaction"} * 20
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_accountRange"} * 500
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_storageRangeAt"} * 500
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_chaindbCompact"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_chaindbProperty"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_dbAncient"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_dbAncients"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_dbGet"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_dumpBlock"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getAccessibleState"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getModifiedAccountsByHash"} *
+        100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getModifiedAccountsByNumber"}
+        * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_preimage"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_printBlock"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_setHead"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_setTrieFlushInterval"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_getTrieFlushInterval"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_intermediateRoots"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_standardTraceBlockToFile"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_standardTraceBadBlockToFile"}
+        * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceBadBlock"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceBlock"} * 497
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceBlockByNumber"} * 497
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceBlockByHash"} * 497
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceBlockFromFile"} * 497
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceCall"} * 309
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceChain"} * 100
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="debug_traceTransaction"} * 309
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="txpool_content"} * 800
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="txpool_contentFrom"} * 800
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="txpool_inspect"} * 800
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="txpool_status"} * 800
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="net_listening"} * 5
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="net_peerCount"} * 5
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="net_version"} * 5
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="web3_clientVersion"} * 15
+      record: subway_rpc_cu_counter
+    - expr: subway_rpc_calls_finished{method="web3_sha3"} * 15
+      record: subway_rpc_cu_counter

scripts/gen-record-rules.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from pathlib import Path
+
+import yaml
+
+DEFAULT_CU = 2
+CU_METRIC = "subway_rpc_calls_finished"
+
+SOURCE_ROOT = Path(__file__).parent.parent
+
+parser = ArgumentParser(
+    description="Generate record rules for a given rpc config",
+    formatter_class=ArgumentDefaultsHelpFormatter,
+)
+parser.add_argument(
+    "config",
+    type=Path,
+    nargs="?",
+    default=SOURCE_ROOT / "rpc_configs/geth.yml",
+    help="Path to the rpc config file",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    type=Path,
+    required=False,
+    help="Path to the output directory",
+)
+parser.add_argument(
+    "-n",
+    "--namespace",
+    type=str,
+    default="proxies",
+    help="Namespace for the generated rules",
+)
+
+# parser.add_argument(
+#     "--vm",
+#     action="store_true",
+#     help="Generate VMRules instead of PrometheusRule",
+# )
+
+parser.add_argument(
+    "--prom",
+    action="store_true",
+    help="Generate PrometheusRule instead of VMRule",
+)
+
+
+def main():
+    args = parser.parse_args()
+    cfgPath: Path = args.config
+    rpcs = yaml.safe_load(cfgPath.read_text())
+    if not args.output:
+        args.output = SOURCE_ROOT / "monitoring" / f"rules.cu.{cfgPath.name}"
+    output = args.output
+    rules = []
+
+    for m in rpcs["methods"]:
+        method = m["method"]
+        weight = m.get("rate_limit_weight", DEFAULT_CU)
+        rules.append(
+            {
+                "record": "subway_rpc_cu_counter",
+                "expr": f'{CU_METRIC}{{method="{method}"}} * {weight}',
+            }
+        )
+
+    manifest = {
+        "apiVersion": "operator.victoriametrics.com/v1beta1",
+        "kind": "VMRule",
+        "metadata": {
+            "name": f"subway-cu-record-{cfgPath.stem}",
+            "namespace": args.namespace,
+        },
+        "spec": {"groups": [{"name": f"subway-cu-record-{cfgPath.stem}.rules", "rules": rules}]},
+    }
+    if args.prom:
+        manifest["apiVersion"] = "monitoring.coreos.com/v1"
+        manifest["kind"] = "PrometheusRule"
+    # if args.vm:
+    #     manifest["apiVersion"] = "operator.victoriametrics.com/v1beta1"
+    #     manifest["kind"] = "VMRule"
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(yaml.dump(manifest))
+
+
+if __name__ == "__main__":
+    main()