diff --git a/.github/workflows/ci-test-py.yml b/.github/workflows/ci-test-py.yml new file mode 100644 index 0000000..78ad5ec --- /dev/null +++ b/.github/workflows/ci-test-py.yml @@ -0,0 +1,52 @@ +name: ci-test-py +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + py-pip-ai-sentryflow: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: check Python pip3 requirements + run: | + pip install -r requirements.txt + working-directory: ai-engine + + py-lint-ai-sentryflow: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + working-directory: ai-engine + + - name: Lint with Ruff + run: | + pip install ruff + ruff --output-format=github . + continue-on-error: true + working-directory: ai-engine + + py-pep8-ai-sentryflow: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: 'Run PEP8' + uses: quentinguidee/pep8-action@v1 + with: + arguments: '--max-line-length=120' \ No newline at end of file diff --git a/.github/workflows/sentryflow-pr-checks.yml b/.github/workflows/sentryflow-pr-checks.yml index 16c5f39..22b844e 100644 --- a/.github/workflows/sentryflow-pr-checks.yml +++ b/.github/workflows/sentryflow-pr-checks.yml @@ -24,7 +24,12 @@ jobs: echo "tag=tmp" >> $GITHUB_OUTPUT fi - - name: Build Docker Image + - name: Build SentryFlow Docker Image working-directory: ./sentryflow run: | make TAG=${{ steps.tag.outputs.tag }} image + + - name: Build SentryFlow AI Engine Docker Image + working-directory: ./ai-engine + run: | + make TAG=${{ steps.tag.outputs.tag }} build diff --git a/.github/workflows/sentryflow-release-image.yml b/.github/workflows/sentryflow-release-image.yml index a32559c..678297a 100644 --- a/.github/workflows/sentryflow-release-image.yml +++ b/.github/workflows/sentryflow-release-image.yml @@ -30,11 +30,16 @@ jobs: echo "tag=tmp" >> $GITHUB_OUTPUT fi - - name: Build Docker Image + - name: Build SentryFlow Docker Image working-directory: ./sentryflow run: | make TAG=${{ steps.tag.outputs.tag }} image + - name: Build SentryFlow AI Engine Docker Image + working-directory: ./ai-engine + run: | + make TAG=${{ steps.tag.outputs.tag }} build + # - name: Push Docker Image # run: | # docker push boanlab/sentryflow:${{ steps.tag.outputs.tag }} diff --git a/.gitignore b/.gitignore index df16c5c..e43b0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ .DS_Store -go.work -go.work.sum \ No newline at end of file diff --git a/README.md b/README.md index bc7ee81..4626c7f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ - # SentryFlow -[![SentryFlow Docker Build](https://github.com/5GSEC/sentryflow/actions/workflows/sentryflow-release-image.yml/badge.svg)](https://github.com/5GSEC/sentryflow/actions/workflows/sentryflow-release-image.yml) [![CI Test](https://github.com/5GSEC/sentryflow/actions/workflows/ci-test-go.yml/badge.svg)](https://github.com/5GSEC/sentryflow/actions/workflows/ci-test-go.yml) +[![SentryFlow Docker Build](https://github.com/5gsec/SentryFlow/actions/workflows/sentryflow-release-image.yml/badge.svg)](https://github.com/5gsec/SentryFlow/actions/workflows/sentryflow-release-image.yml) [![CI Test](https://github.com/5gsec/SentryFlow/actions/workflows/ci-test-go.yml/badge.svg)](https://github.com/5gsec/SentryFlow/actions/workflows/ci-test-go.yml) [![ci-test-py](https://github.com/5gsec/SentryFlow/actions/workflows/ci-test-py.yml/badge.svg)](https://github.com/5gsec/SentryFlow/actions/workflows/ci-test-py.yml) SentryFlow is a cloud-native system for API observability and security, specializing in log collection, metric production, and data exportation. ## Architecture Overview -![Sentryflow Overview](docs/sentryflow_overview.png) +![SentryFlow_Overview](docs/sentryflow_overview.png) ### Features - Generation of API Access Logs diff --git a/ai-engine/.dockerignore b/ai-engine/.dockerignore new file mode 100644 index 0000000..23ca759 --- /dev/null +++ b/ai-engine/.dockerignore @@ -0,0 +1,6 @@ +.idea +.git +.gitignore +protobuf +Dockerfile +__pycache__/ \ No newline at end of file diff --git a/ai-engine/.gitignore b/ai-engine/.gitignore new file mode 100644 index 0000000..533d889 --- /dev/null +++ b/ai-engine/.gitignore @@ -0,0 +1,3 @@ +.idea/ +__pycache__/ +protobuf/ \ No newline at end of file diff --git a/ai-engine/Dockerfile b/ai-engine/Dockerfile new file mode 100644 index 0000000..1e40850 --- /dev/null +++ b/ai-engine/Dockerfile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Dockerfile +FROM ubuntu:latest + +RUN apt-get update && apt-get -y install python3 python3-pip wget git + +RUN git clone https://github.com/isu-kim/stringlifier.git +WORKDIR ./stringlifier +RUN pip install . + +RUN mkdir /app +WORKDIR /app +COPY /ai-engine . + +# Build protobuf for Python +RUN pip install grpcio grpcio-tools +RUN mkdir protobuf/ +COPY /protobuf ./protobuf + +# Due to python import bugs, we have to compile protoc using this command +# Refer to https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-261621112 for more information on this +RUN python3 -m grpc_tools.protoc --python_out=. --pyi_out=. --grpc_python_out=. -I=. protobuf/sentryflow_metrics.proto + +WORKDIR /app +RUN pip install -r requirements.txt + +CMD ["python3", "ai-engine.py"] diff --git a/ai-engine/Makefile b/ai-engine/Makefile new file mode 100644 index 0000000..c65acdb --- /dev/null +++ b/ai-engine/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 + +IMAGE_NAME = 5gsec/sentryflow-ai-engine +TAG = v0.1 + +.PHONY: build + +build: + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../ diff --git a/ai-engine/ai-engine.py b/ai-engine/ai-engine.py new file mode 100644 index 0000000..39efa74 --- /dev/null +++ b/ai-engine/ai-engine.py @@ -0,0 +1,94 @@ +import os +import grpc + +from stringlifier.api import Stringlifier +from concurrent import futures + +from protobuf import sentryflow_metrics_pb_grpc +from protobuf import sentryflow_metrics_pb + + +class HandlerServer: + """ + Class for gRPC Servers + """ + def __init__(self): + try: + self.listen_addr = os.environ["AI_ENGINE_ADDRESS"] + except KeyError: + self.listen_addr = "0.0.0.0:5000" + + self.server = None + self.grpc_servers = list() + + def init_grpc_servers(self): + """ + init_grpc_servers method that initializes and registers gRPC servers + :return: None + """ + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + self.grpc_servers.append(APIClassificationServer()) # @todo: make this configurable + + grpc_server: GRPCServer + for grpc_server in self.grpc_servers: + grpc_server.register(self.server) + + def serve(self): + """ + serve method that starts serving gRPC servers, this is blocking function. + :return: None + """ + self.server.add_insecure_port(self.listen_addr) + + print("[INFO] Starting to serve on {}".format(self.listen_addr)) + self.server.start() + self.server.wait_for_termination() + + +class GRPCServer: + """ + Abstract class for an individual gRPC Server + """ + def register(self, server): + """ + register method that registers gRPC service to target server + :param server: The server + :return: None + """ + pass + + +class APIClassificationServer(sentryflow_metrics_pb2_grpc.SentryFlowMetricsServicer, GRPCServer): + """ + Class for API Classification Server using Stringlifier + """ + + def __init__(self): + self.stringlifier = Stringlifier() + print("[Init] Successfully initialized APIClassificationServer") + + def register(self, server): + sentryflow_metrics_pb2_grpc.add_SentryFlowMetricsServicer_to_server(self, server) + + def GetAPIClassification(self, request_iterator, context): + """ + GetAPIClassification method that runs multiple API ML Classification at once + :param request_iterator: The requests + :param context: The context + :return: The results + """ + + for req in request_iterator: + paths = req.paths + ml_results = self.stringlifier(paths) + print("{} -> {}".format(paths, ml_results)) + + results = [sentryflow_metrics_pb2.APIClassificationSingleResponse(merged=ml_result, fields=[]) for ml_result + in ml_results] + yield sentryflow_metrics_pb2.APIClassificationResponse(response=results) + + +if __name__ == '__main__': + hs = HandlerServer() + hs.init_grpc_servers() + hs.serve() diff --git a/ai-engine/requirements.txt b/ai-engine/requirements.txt new file mode 100644 index 0000000..7c37043 Binary files /dev/null and b/ai-engine/requirements.txt differ diff --git a/contribution/vagrant/setup.sh b/contribution/vagrant/setup.sh index da111f7..fe59d97 100755 --- a/contribution/vagrant/setup.sh +++ b/contribution/vagrant/setup.sh @@ -4,10 +4,18 @@ git clone https://github.com/boanlab/tools.git # Install Docker -bash tools/containers/install-docker.sh +bash tools/containers/install-containerd.sh # Install Kubeadm -bash tools/kubernetes/install-kubeadm.sh +sudo apt-get update +sudo apt-get install -y apt-transport-https ca-certificates curl gpg +curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.24/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.24/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list +sudo sysctl -w net.ipv4.ip_forward=1 +sudo swapoff -a +sudo apt-get update +sudo apt-get install -y kubelet kubeadm kubectl +sudo apt-mark hold kubelet kubeadm kubectl # Disable Swap sudo swapoff -a diff --git a/deployments/sentryflow.yaml b/deployments/sentryflow.yaml index 3284d75..0c9a378 100644 --- a/deployments/sentryflow.yaml +++ b/deployments/sentryflow.yaml @@ -8,6 +8,43 @@ metadata: pod-security.kubernetes.io/enforce: privileged pod-security.kubernetes.io/warn: privileged --- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-engine + namespace: sentryflow +spec: + replicas: 1 + selector: + matchLabels: + app: ai-engine + template: + metadata: + labels: + app: ai-engine + spec: + containers: + - name: sentryflow + image: 5gsec/sentryflow-ai-engine:v0.1 + ports: + - containerPort: 5000 + protocol: TCP + name: grpc-sentryflow +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-engine + namespace: sentryflow +spec: + selector: + app: ai-engine + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 + name: grpc-sentryflow +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -54,7 +91,7 @@ spec: serviceAccountName: sa-sentryflow containers: - name: sentryflow - image: 5gsec/sentryflow:v0.0.1 + image: 5gsec/sentryflow:v0.1 ports: - containerPort: 4317 protocol: TCP diff --git a/protobuf/.gitignore b/protobuf/.gitignore index cd1cbd6..787cab2 100644 --- a/protobuf/.gitignore +++ b/protobuf/.gitignore @@ -1,2 +1,3 @@ .idea/ -*.pb.go \ No newline at end of file +*.pb.go +*.tar \ No newline at end of file diff --git a/protobuf/Makefile b/protobuf/Makefile index eea513e..945e295 100644 --- a/protobuf/Makefile +++ b/protobuf/Makefile @@ -1,4 +1,4 @@ -PROTO:=sentryflow.proto +PROTO:=sentryflow.proto sentryflow_metrics.proto PBGO:=$(PROTO:.proto=.pb.go) .PHONY: build @@ -13,4 +13,4 @@ go.sum: go.mod .PHONY: clean clean: - rm -f go.sum *.pb.go \ No newline at end of file + rm -f go.sum *.pb.go diff --git a/protobuf/go.mod b/protobuf/go.mod index 986067f..6d856bd 100644 --- a/protobuf/go.mod +++ b/protobuf/go.mod @@ -3,14 +3,14 @@ module github.com/5GSEC/sentryflow/protobuf go 1.19 require ( - google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 + google.golang.org/grpc v1.61.1 + google.golang.org/protobuf v1.32.0 ) require ( github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect ) diff --git a/protobuf/go.sum b/protobuf/go.sum index 0173cce..a358d18 100644 --- a/protobuf/go.sum +++ b/protobuf/go.sum @@ -2,19 +2,19 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= +google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/protobuf/sentryflow.proto b/protobuf/sentryflow.proto index 9421b5a..d8b2281 100644 --- a/protobuf/sentryflow.proto +++ b/protobuf/sentryflow.proto @@ -38,7 +38,19 @@ message APIMetric { // @todo: add some more metrics here } +message Metric { + string type = 1; + string key = 2; + string value = 3; +} + +message EnvoyMetric { + string identifier = 1; + repeated Metric metric = 2; +} + service SentryFlow { rpc GetLog(ClientInfo) returns (stream APILog); rpc GetAPIMetrics(ClientInfo) returns (APIMetric); + rpc GetEnvoyMetrics(ClientInfo) returns (stream EnvoyMetric); } diff --git a/protobuf/sentryflow_metrics.proto b/protobuf/sentryflow_metrics.proto new file mode 100644 index 0000000..46c3e1f --- /dev/null +++ b/protobuf/sentryflow_metrics.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package protobuf; + +option go_package = "sentryflow/protobuf"; + +message APIClassificationRequest { + string path = 1; +} + +message APIClassificationResponse { + string merged = 1; + repeated string fields = 2; +} + +service SentryFlowMetrics { + rpc GetAPIClassification(stream APIClassificationRequest) returns (stream APIClassificationResponse); +} diff --git a/Dockerfile.log-client b/sentryflow-clients/log-client/Dockerfile similarity index 88% rename from Dockerfile.log-client rename to sentryflow-clients/log-client/Dockerfile index f0ff44f..e0f7f5b 100644 --- a/Dockerfile.log-client +++ b/sentryflow-clients/log-client/Dockerfile @@ -2,26 +2,24 @@ ### Builder -FROM golang:1.22.0-alpine3.19 as builder +FROM golang:1.19-alpine3.17 as builder RUN apk --no-cache update RUN apk add --no-cache git clang llvm make gcc protobuf -RUN go install github.com/golang/protobuf/protoc-gen-go@latest -RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest RUN mkdir /app RUN mkdir /protobuf WORKDIR /protobuf -COPY /protobuf . -RUN go mod tidy -RUN make build +COPY /protobuf . WORKDIR /app + COPY /sentryflow-clients/log-client . -RUN go mod tidy +RUN go install github.com/golang/protobuf/protoc-gen-go@latest +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest RUN go build -o log-client ### Make executable image diff --git a/sentryflow-clients/log-client/Makefile b/sentryflow-clients/log-client/Makefile index d941259..7b8bfdb 100644 --- a/sentryflow-clients/log-client/Makefile +++ b/sentryflow-clients/log-client/Makefile @@ -6,7 +6,7 @@ TAG = v0.1 .PHONY: build build: - docker build -t $(IMAGE_NAME):$(TAG) -f ../../Dockerfile.log-client ../../ + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../../ .PHONY: clean diff --git a/sentryflow-clients/log-client/main.go b/sentryflow-clients/log-client/main.go index 27767e9..05a1026 100644 --- a/sentryflow-clients/log-client/main.go +++ b/sentryflow-clients/log-client/main.go @@ -11,6 +11,8 @@ import ( "log" "log-client/common" "os" + "os/signal" + "syscall" "sentryflow/protobuf" ) @@ -32,7 +34,7 @@ func main() { defer conn.Close() // Start serving gRPC server - log.Printf("[gRPC] Successfully connected to %s", addr) + log.Printf("[gRPC] Successfully connected to %s for AccessLog", addr) // Create a client for the SentryFlow service client := protobuf.NewSentryFlowClient(conn) @@ -48,19 +50,58 @@ func main() { } // Contact the server and print out its response - stream, err := client.GetLog(context.Background(), clientInfo) + accessLogStream, err := client.GetLog(context.Background(), clientInfo) + metricStream, err := client.GetEnvoyMetrics(context.Background(), clientInfo) if err != nil { log.Fatalf("could not get log: %v", err) } + done := make(chan struct{}) + + go accessLogRoutine(accessLogStream, done) + go metricRoutine(metricStream, done) + + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + <-signalChan + + close(done) +} + +func accessLogRoutine(stream protobuf.SentryFlow_GetLogClient, done chan struct{}) { for { - data, err := stream.Recv() - if err == io.EOF { - break + select { + default: + data, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive log: %v", err) + } + log.Printf("[Client] Received log: %v", data) + case <-done: + return } - if err != nil { - log.Fatalf("failed to receive log: %v", err) + } +} + +func metricRoutine(stream protobuf.SentryFlow_GetEnvoyMetricsClient, done chan struct{}) { + for { + select { + default: + data, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive metric: %v", err) + } + log.Printf("[Client] Received metric: %v", data) + case <-done: + return } - log.Printf("[Client] Received log: %v", data) } } diff --git a/Dockerfile.mongo-client b/sentryflow-clients/mongo-client/Dockerfile similarity index 90% rename from Dockerfile.mongo-client rename to sentryflow-clients/mongo-client/Dockerfile index 4d744f8..1a0d3ca 100644 --- a/Dockerfile.mongo-client +++ b/sentryflow-clients/mongo-client/Dockerfile @@ -2,26 +2,24 @@ ### Builder -FROM golang:1.22.0-alpine3.19 as builder +FROM golang:1.19-alpine3.17 as builder RUN apk --no-cache update RUN apk add --no-cache git clang llvm make gcc protobuf -RUN go install github.com/golang/protobuf/protoc-gen-go@latest -RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest RUN mkdir /app RUN mkdir /protobuf WORKDIR /protobuf + COPY /protobuf . -RUN go mod tidy -RUN make build WORKDIR /app COPY /sentryflow-clients/mongo-client . - +RUN go install github.com/golang/protobuf/protoc-gen-go@latest +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest RUN go build -o mongo-client ### Make executable image diff --git a/sentryflow-clients/mongo-client/Makefile b/sentryflow-clients/mongo-client/Makefile index 03d92a3..c1cba29 100644 --- a/sentryflow-clients/mongo-client/Makefile +++ b/sentryflow-clients/mongo-client/Makefile @@ -6,7 +6,7 @@ TAG = v0.1 .PHONY: build build: - docker build -t $(IMAGE_NAME):$(TAG) -f ../../Dockerfile.mongo-client ../../ + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../../ .PHONY: clean diff --git a/sentryflow-clients/mongo-client/db/dbHandler.go b/sentryflow-clients/mongo-client/db/dbHandler.go index ae94049..b63edac 100644 --- a/sentryflow-clients/mongo-client/db/dbHandler.go +++ b/sentryflow-clients/mongo-client/db/dbHandler.go @@ -15,11 +15,12 @@ import ( ) type Handler struct { - client *mongo.Client - database *mongo.Database - collection *mongo.Collection - cancel context.CancelFunc - dbURL string + client *mongo.Client + database *mongo.Database + alCollection *mongo.Collection + metricsCollection *mongo.Collection + cancel context.CancelFunc + dbURL string } var Manager *Handler @@ -55,7 +56,8 @@ func New() (*Handler, error) { // Create 'sentryflow' database and 'api-logs' collection h.database = h.client.Database("sentryflow") - h.collection = h.database.Collection("api-logs") + h.alCollection = h.database.Collection("api-logs") + h.metricsCollection = h.database.Collection("metrics") Manager = &h return &h, nil @@ -70,8 +72,17 @@ func (h *Handler) Disconnect() { return } -func (h *Handler) InsertData(data *protobuf.APILog) error { - _, err := h.collection.InsertOne(context.Background(), data) +func (h *Handler) InsertAl(data *protobuf.APILog) error { + _, err := h.alCollection.InsertOne(context.Background(), data) + if err != nil { + return err + } + + return nil +} + +func (h *Handler) InsertMetrics(data *protobuf.EnvoyMetric) error { + _, err := h.metricsCollection.InsertOne(context.Background(), data) if err != nil { return err } diff --git a/sentryflow-clients/mongo-client/main.go b/sentryflow-clients/mongo-client/main.go index d41651f..150f9d7 100644 --- a/sentryflow-clients/mongo-client/main.go +++ b/sentryflow-clients/mongo-client/main.go @@ -11,9 +11,55 @@ import ( "mongo-client/common" "mongo-client/db" "os" + "os/signal" protobuf "sentryflow/protobuf" + "syscall" ) +func accessLogRoutine(stream protobuf.SentryFlow_GetLogClient, done chan struct{}) { + for { + select { + default: + data, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive log: %v", err) + } + log.Printf("[Client] Inserting log") + err = db.Manager.InsertAl(data) + if err != nil { + log.Printf("[Client] Failed to insert log: %v", err) + } + case <-done: + return + } + } +} + +func metricRoutine(stream protobuf.SentryFlow_GetEnvoyMetricsClient, done chan struct{}) { + for { + select { + default: + data, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to receive metric: %v", err) + } + log.Printf("[Client] Inserting metric") + err = db.Manager.InsertMetrics(data) + if err != nil { + log.Printf("[Client] Failed to insert metric: %v", err) + } + case <-done: + return + } + } +} + func main() { // Init DB _, err := db.New() @@ -53,24 +99,22 @@ func main() { HostName: hostname, } - // Contact the server and print out its response. - stream, err := client.GetLog(context.Background(), clientInfo) + // Contact the server and print out its response + accessLogStream, err := client.GetLog(context.Background(), clientInfo) + metricStream, err := client.GetEnvoyMetrics(context.Background(), clientInfo) if err != nil { log.Fatalf("could not get log: %v", err) } - for { - data, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Fatalf("failed to receive log: %v", err) - } + done := make(chan struct{}) - err = db.Manager.InsertData(data) - if err != nil { - log.Printf("[DB] Failed to store data to DB: %v", err) - } - } + go accessLogRoutine(accessLogStream, done) + go metricRoutine(metricStream, done) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + <-signalChan + + close(done) } diff --git a/Dockerfile b/sentryflow/Dockerfile similarity index 86% rename from Dockerfile rename to sentryflow/Dockerfile index a509f41..ce023b4 100644 --- a/Dockerfile +++ b/sentryflow/Dockerfile @@ -2,10 +2,11 @@ ### Builder -FROM golang:1.22.0-alpine3.19 as builder +FROM golang:1.19-alpine3.17 as builder RUN apk --no-cache update RUN apk add --no-cache git clang llvm make gcc protobuf make +RUN apk add --update alpine-sdk RUN go install github.com/golang/protobuf/protoc-gen-go@latest RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest @@ -21,6 +22,7 @@ WORKDIR /app COPY /sentryflow . RUN go mod tidy +RUN export CGO_ENABLED=1; export CC=gcc; RUN go build -o sentryflow ### Make executable image diff --git a/sentryflow/Makefile b/sentryflow/Makefile index 3831ca6..1930584 100644 --- a/sentryflow/Makefile +++ b/sentryflow/Makefile @@ -11,10 +11,14 @@ build: .PHONY: image image: - docker build -t $(IMAGE_NAME):$(TAG) -f ../Dockerfile ../ + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../ -.PHONY: clean -clean: +.PHONY: clean-build +clean-build: + rm -f sentryflow + +.PHONY: clean-image +clean-image: docker rmi $(IMAGE_NAME):$(TAG) .PHONY: run diff --git a/sentryflow/collector/collectorHandler.go b/sentryflow/collector/collectorHandler.go new file mode 100644 index 0000000..6251d72 --- /dev/null +++ b/sentryflow/collector/collectorHandler.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "errors" + "fmt" + cfg "github.com/5GSEC/sentryflow/config" + "google.golang.org/grpc" + "log" + "net" + "sync" +) + +// Ch global reference for Collector Handler +var Ch *Handler + +// init Function +func init() { + Ch = NewCollectorHandler() +} + +// Handler Structure +type Handler struct { + collectors []collectorInterface + + listener net.Listener + grpcServer *grpc.Server + + wg sync.WaitGroup +} + +// NewCollectorHandler Function +func NewCollectorHandler() *Handler { + ch := &Handler{ + collectors: make([]collectorInterface, 0), + } + + return ch +} + +// InitGRPCServer Function +func (h *Handler) InitGRPCServer() error { + listenAddr := fmt.Sprintf("%s:%s", cfg.GlobalCfg.OtelGRPCListenAddr, cfg.GlobalCfg.OtelGRPCListenPort) + + // Start listening + lis, err := net.Listen("tcp", listenAddr) + if err != nil { + msg := fmt.Sprintf("unable to listen at %s: %v", listenAddr, err) + return errors.New(msg) + } + + // Create gRPC Server, register services + server := grpc.NewServer() + + h.listener = lis + h.grpcServer = server + + // initialize collectors + err = h.initCollectors() + if err != nil { + log.Printf("[Collector] Unable to initialize collector: %v", err) + } + + // register services + h.registerServices() + + log.Printf("[Collector] Server listening at %s", listenAddr) + return nil +} + +// initCollectors Function +func (h *Handler) initCollectors() error { + // @todo make configuration determine which collector to start or not + h.collectors = append(h.collectors, newOtelLogServer()) + h.collectors = append(h.collectors, newEnvoyMetricsServer()) + h.collectors = append(h.collectors, newEnvoyAccessLogsServer()) + + return nil +} + +// registerServices Function +func (h *Handler) registerServices() { + for _, col := range h.collectors { + col.registerService(h.grpcServer) + log.Printf("[Collector] Successfully registered services") + } +} + +// Serve Function +func (h *Handler) Serve() error { + log.Printf("[Collector] Starting gRPC server") + return h.grpcServer.Serve(h.listener) +} + +// Stop Function +func (h *Handler) Stop() { + log.Printf("[Collector] Stopped gRPC server") + h.grpcServer.GracefulStop() +} diff --git a/sentryflow/collector/envoy.go b/sentryflow/collector/envoy.go new file mode 100644 index 0000000..2146cd3 --- /dev/null +++ b/sentryflow/collector/envoy.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "fmt" + "github.com/5GSEC/sentryflow/core" + "github.com/5GSEC/sentryflow/protobuf" + envoyAls "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" + envoyMetrics "github.com/envoyproxy/go-control-plane/envoy/service/metrics/v3" + "google.golang.org/grpc" + "io" + "log" +) + +// EnvoyMetricsServer Structure +type EnvoyMetricsServer struct { + envoyMetrics.UnimplementedMetricsServiceServer + collectorInterface +} + +// newEnvoyMetricsServer Function +func newEnvoyMetricsServer() *EnvoyMetricsServer { + ret := &EnvoyMetricsServer{} + return ret +} + +// registerService Function +func (ems *EnvoyMetricsServer) registerService(server *grpc.Server) { + envoyMetrics.RegisterMetricsServiceServer(server, ems) +} + +// StreamMetrics Function +func (ems *EnvoyMetricsServer) StreamMetrics(stream envoyMetrics.MetricsService_StreamMetricsServer) error { + event, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + log.Printf("[Envoy] Something went on wrong when receiving event: %v", err) + return err + } + + err = event.ValidateAll() + if err != nil { + log.Printf("[Envoy] Failed to validate stream: %v", err) + } + + // @todo parse this event entry into our format + identifier := event.GetIdentifier() + identifier.GetNode().GetMetadata() + + if identifier != nil { + log.Printf("[Envoy] Received EnvoyMetric - ID: %s, %s", identifier.GetNode().GetId(), identifier.GetNode().GetCluster()) + + nodeID := identifier.GetNode().GetId() + cluster := identifier.GetNode().GetCluster() + + curIdentifier := fmt.Sprintf("%s, %s", nodeID, cluster) + envoyMetric := &protobuf.EnvoyMetric{ + Identifier: curIdentifier, + Metric: []*protobuf.Metric{}, + } + + for _, metric := range event.GetEnvoyMetrics() { + metricType := metric.GetType().String() + metricName := metric.GetName() + tempMetrics := metric.GetMetric() + metrics := fmt.Sprintf("%s", tempMetrics) + + curMetric := &protobuf.Metric{ + Type: metricType, + Key: metricName, + Value: metrics, + } + + envoyMetric.Metric = append(envoyMetric.Metric, curMetric) + } + + core.Lh.InsertLog(envoyMetric) + } + + return nil +} + +// EnvoyAccessLogsServer Structure +type EnvoyAccessLogsServer struct { + envoyAls.UnimplementedAccessLogServiceServer + collectorInterface +} + +// newEnvoyAccessLogsServer Function +func newEnvoyAccessLogsServer() *EnvoyAccessLogsServer { + ret := &EnvoyAccessLogsServer{} + return ret +} + +// registerService Function +func (eas *EnvoyAccessLogsServer) registerService(server *grpc.Server) { + envoyAls.RegisterAccessLogServiceServer(server, eas) +} + +// StreamAccessLogs Function +func (eas *EnvoyAccessLogsServer) StreamAccessLogs(stream envoyAls.AccessLogService_StreamAccessLogsServer) error { + for { + event, err := stream.Recv() + if err == io.EOF { + return nil + } + + if err != nil { + log.Printf("[Envoy] Something went on wrong when receiving event: %v", err) + return err + } + + // Check HTTP logs + if event.GetHttpLogs() != nil { + for _, entry := range event.GetHttpLogs().LogEntry { + envoyAccessLog := core.GenerateAccessLogsFromEnvoy(entry) + core.Lh.InsertLog(envoyAccessLog) + } + } + } +} diff --git a/sentryflow/collector/interface.go b/sentryflow/collector/interface.go new file mode 100644 index 0000000..154d83c --- /dev/null +++ b/sentryflow/collector/interface.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import "google.golang.org/grpc" + +// collectorInterface Interface +type collectorInterface interface { + registerService(server *grpc.Server) +} diff --git a/sentryflow/collector/opentelemetry.go b/sentryflow/collector/opentelemetry.go new file mode 100644 index 0000000..6b641d8 --- /dev/null +++ b/sentryflow/collector/opentelemetry.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "context" + "github.com/5GSEC/sentryflow/core" + otelLogs "go.opentelemetry.io/proto/otlp/collector/logs/v1" + "google.golang.org/grpc" +) + +// OtelLogServer structure +type OtelLogServer struct { + otelLogs.UnimplementedLogsServiceServer + collectorInterface +} + +// newOtelLogServer Function +func newOtelLogServer() *OtelLogServer { + ret := &OtelLogServer{} + return ret +} + +// registerService Function +func (ols *OtelLogServer) registerService(server *grpc.Server) { + otelLogs.RegisterLogsServiceServer(server, ols) +} + +// Export Function +func (ols *OtelLogServer) Export(_ context.Context, req *otelLogs.ExportLogsServiceRequest) (*otelLogs.ExportLogsServiceResponse, error) { + // This is for Log.Export in OpenTelemetry format + als := core.GenerateAccessLogsFromOtel(req.String()) + + for _, al := range als { + core.Lh.InsertLog(al) + } + + // For now, we will not consider partial success + ret := otelLogs.ExportLogsServiceResponse{ + PartialSuccess: nil, + } + + return &ret, nil +} diff --git a/sentryflow/config/config.go b/sentryflow/config/config.go index dd2ac60..98685a2 100644 --- a/sentryflow/config/config.go +++ b/sentryflow/config/config.go @@ -12,8 +12,8 @@ import ( "github.com/spf13/viper" ) -// NumbatConfig structure -type NumbatConfig struct { +// SentryFlowConfig structure +type SentryFlowConfig struct { OtelGRPCListenAddr string // IP address to use for OTEL gRPC OtelGRPCListenPort string // Port to use for OTEL gRPC @@ -23,21 +23,36 @@ type NumbatConfig struct { PatchNamespace bool // Enable/Disable patching namespace for Istio injection PatchRestartDeployments bool // Enable/Disable restarting deployments after patching - Debug bool // Enable/Disable SentryFlow debug mode + AIEngineService string + AIEngineBatchSize int + + MetricsDBFileName string // String value of MetricsDB file (sqlite3 db file) + + CollectorEnableOpenTelemetry bool // Enable/Disable OpenTelemetry Collector + Debug bool // Enable/Disable SentryFlow debug mode } // GlobalCfg Global configuration for SentryFlow -var GlobalCfg NumbatConfig +var GlobalCfg SentryFlowConfig + +// init Function +func init() { + _ = LoadConfig() +} // Config const const ( - OtelGRPCListenAddr string = "otelGRPCListenAddr" - OtelGRPCListenPort string = "otelGRPCListenPort" - CustomExportListenAddr string = "customExportListenAddr" - CustomExportListenPort string = "customExportListenPort" - PatchNamespace string = "patchNamespace" - PatchRestartDeployments string = "patchRestartDeployments" - Debug string = "debug" + OtelGRPCListenAddr string = "otelGRPCListenAddr" + OtelGRPCListenPort string = "otelGRPCListenPort" + CustomExportListenAddr string = "customExportListenAddr" + CustomExportListenPort string = "customExportListenPort" + PatchNamespace string = "patchNamespace" + PatchRestartDeployments string = "patchRestartDeployments" + AIEngineService string = "aiEngineService" + AIEngineBatchSize string = "aiEngineBatchSize" + MetricsDBFileName string = "metricsDBFileName" + CollectorEnableOpenTelemetry string = "collectorEnableOpenTelemetry" + Debug string = "debug" ) func readCmdLineParams() { @@ -47,6 +62,10 @@ func readCmdLineParams() { customExportListenPortStr := flag.String(CustomExportListenPort, "8080", "Custom export gRPC server listen port") patchNamespaceB := flag.Bool(PatchNamespace, false, "Enable/Disable patching Istio injection to all namespaces") patchRestartDeploymentsB := flag.Bool(PatchRestartDeployments, false, "Enable/Disable restarting deployments in all namespaces") + aiEngineServiceStr := flag.String(AIEngineService, "ai-engine.sentryflow.svc.cluster.local", "Service address for SentryFlow AI Engine") + aiEngineBatchSizeInt := flag.Int(AIEngineBatchSize, 5, "Batch size fo SentryFlow AI Engine") + metricsDBFileNameStr := flag.String(MetricsDBFileName, "/etc/sentryflow/metrics.db", "File name for local metrics DB") + collectorEnableOpenTelemetryB := flag.Bool(CollectorEnableOpenTelemetry, true, "Enable/Disable OpenTelemetry Collector") configDebugB := flag.Bool(Debug, false, "Enable/Disable debugging mode using logs") var flags []string @@ -64,6 +83,10 @@ func readCmdLineParams() { viper.SetDefault(CustomExportListenPort, *customExportListenPortStr) viper.SetDefault(PatchNamespace, *patchNamespaceB) viper.SetDefault(PatchRestartDeployments, *patchRestartDeploymentsB) + viper.SetDefault(AIEngineService, *aiEngineServiceStr) + viper.SetDefault(AIEngineBatchSize, *aiEngineBatchSizeInt) + viper.SetDefault(MetricsDBFileName, *metricsDBFileNameStr) + viper.SetDefault(CollectorEnableOpenTelemetry, *collectorEnableOpenTelemetryB) viper.SetDefault(Debug, *configDebugB) } @@ -76,7 +99,7 @@ func LoadConfig() error { viper.AutomaticEnv() // todo: read configuration from config file - _ = os.Getenv("NUMBAT_CFG") + _ = os.Getenv("SENTRYFLOW_CFG") GlobalCfg.OtelGRPCListenAddr = viper.GetString(OtelGRPCListenAddr) GlobalCfg.OtelGRPCListenPort = viper.GetString(OtelGRPCListenPort) @@ -84,6 +107,10 @@ func LoadConfig() error { GlobalCfg.CustomExportListenPort = viper.GetString(CustomExportListenPort) GlobalCfg.PatchNamespace = viper.GetBool(PatchNamespace) GlobalCfg.PatchRestartDeployments = viper.GetBool(PatchRestartDeployments) + GlobalCfg.AIEngineService = viper.GetString(AIEngineService) + GlobalCfg.AIEngineBatchSize = viper.GetInt(AIEngineBatchSize) + GlobalCfg.MetricsDBFileName = viper.GetString(MetricsDBFileName) + GlobalCfg.CollectorEnableOpenTelemetry = viper.GetBool(CollectorEnableOpenTelemetry) GlobalCfg.Debug = viper.GetBool(Debug) log.Printf("Configuration [%+v]", GlobalCfg) diff --git a/sentryflow/core/k8sHandler.go b/sentryflow/core/k8sHandler.go index 1ffae22..879d9fd 100644 --- a/sentryflow/core/k8sHandler.go +++ b/sentryflow/core/k8sHandler.go @@ -328,6 +328,29 @@ func (kh *K8sHandler) PatchIstioConfigMap() error { return err } + _, eeaExist := meshConfig["enableEnvoyAccessLogService"] + + if eeaExist { + log.Printf("Overwrite the contents of \"enableEnvoyAccessLogService\"") + } + meshConfig["enableEnvoyAccessLogService"] = true + + _, ealExist := meshConfig["defaultConfig"].(map[interface{}]interface{})["envoyAccessLogService"] + if ealExist { + log.Printf("Overwrite the contents of \"defaultConfig.envoyAccessLogService\"") + } + meshConfig["defaultConfig"].(map[interface{}]interface{})["envoyAccessLogService"] = map[string]string{ + "address": "sentryflow.sentryflow.svc.cluster.local:4317", + } + + _, emExist := meshConfig["defaultConfig"].(map[interface{}]interface{})["envoyMetricsService"] + if emExist { + log.Printf("Overwrite the contents of \"defaultConfig.envoyMetricsService\"") + } + meshConfig["defaultConfig"].(map[interface{}]interface{})["envoyMetricsService"] = map[string]string{ + "address": "sentryflow.sentryflow.svc.cluster.local:4317", + } + // Work with defaultProviders.accessLogs dp, exists := meshConfig["defaultProviders"].(map[interface{}]interface{})["accessLogs"] if !exists { // Add defaultProviders.accessLogs if it does not exist diff --git a/sentryflow/core/logHandler.go b/sentryflow/core/logHandler.go index 4be0cd1..ffb6d5e 100644 --- a/sentryflow/core/logHandler.go +++ b/sentryflow/core/logHandler.go @@ -7,6 +7,7 @@ import ( "github.com/5GSEC/sentryflow/metrics" "github.com/5GSEC/sentryflow/protobuf" "github.com/5GSEC/sentryflow/types" + accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" "log" "strconv" "strings" @@ -66,6 +67,8 @@ func (lh *LogHandler) logProcessingRoutine(wg *sync.WaitGroup) { switch l.(type) { case *protobuf.APILog: go processAccessLog(l.(*protobuf.APILog)) + case *protobuf.EnvoyMetric: + go processEnvoyMetric(l.(*protobuf.EnvoyMetric)) } case <-lh.stopChan: @@ -84,8 +87,12 @@ func processAccessLog(al *protobuf.APILog) { metrics.InsertAccessLog(al) } -// GenerateAccessLogs Function -func GenerateAccessLogs(logText string) []*protobuf.APILog { +func processEnvoyMetric(em *protobuf.EnvoyMetric) { + exporter.InsertEnvoyMetric(em) +} + +// GenerateAccessLogsFromOtel Function +func GenerateAccessLogsFromOtel(logText string) []*protobuf.APILog { // @todo this needs more optimization, this code is kind of messy // Create an array of AccessLogs for returning gRPC comm var index int @@ -174,3 +181,50 @@ func GenerateAccessLogs(logText string) []*protobuf.APILog { return ret } + +// GenerateAccessLogsFromEnvoy Function +func GenerateAccessLogsFromEnvoy(entry *accesslogv3.HTTPAccessLogEntry) *protobuf.APILog { + srcInform := entry.GetCommonProperties().GetDownstreamRemoteAddress().GetSocketAddress() + srcIP := srcInform.GetAddress() + srcPort := strconv.Itoa(int(srcInform.GetPortValue())) + src := LookupNetworkedResource(srcIP) + + dstInform := entry.GetCommonProperties().GetUpstreamRemoteAddress().GetSocketAddress() + dstIP := dstInform.GetAddress() + dstPort := strconv.Itoa(int(dstInform.GetPortValue())) + dst := LookupNetworkedResource(dstIP) + + req := entry.GetRequest() + res := entry.GetResponse() + comm := entry.GetCommonProperties() + proto := entry.GetProtocolVersion() + + timeStamp := comm.GetStartTime().Seconds + path := req.GetPath() + method := req.GetRequestMethod().String() + protocolName := proto.String() + resCode := res.GetResponseCode().GetValue() + + envoyAccessLog := &protobuf.APILog{ + TimeStamp: strconv.FormatInt(timeStamp, 10), + Id: 0, // do 0 for now, we are going to write it later + SrcNamespace: src.Namespace, + SrcName: src.Name, + SrcLabel: src.Labels, + SrcIP: srcIP, + SrcPort: srcPort, + SrcType: types.K8sResourceTypeToString(src.Type), + DstNamespace: dst.Namespace, + DstName: dst.Name, + DstLabel: dst.Labels, + DstIP: dstIP, + DstPort: dstPort, + DstType: types.K8sResourceTypeToString(dst.Type), + Protocol: protocolName, + Method: method, + Path: path, + ResponseCode: int32(resCode), + } + + return envoyAccessLog +} diff --git a/sentryflow/core/otelHandler.go b/sentryflow/core/otelHandler.go deleted file mode 100644 index 851328e..0000000 --- a/sentryflow/core/otelHandler.go +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package core - -import ( - "context" - "errors" - "fmt" - cfg "github.com/5GSEC/sentryflow/config" - otelLogs "go.opentelemetry.io/proto/otlp/collector/logs/v1" - "google.golang.org/grpc" - "log" - "net" - "sync" -) - -// Oh Global reference for OtelHandler -var Oh *OtelHandler -var olh *OtelLogServer - -// init Function -func init() { - Oh = NewOtelHandler() - olh = NewOtelLogServer() -} - -// OtelHandler Structure -type OtelHandler struct { - stopChan chan struct{} - - listener net.Listener - gRPCServer *grpc.Server -} - -// NewOtelHandler Function -func NewOtelHandler() *OtelHandler { - oh := &OtelHandler{ - stopChan: make(chan struct{}), - } - - return oh -} - -// InitOtelServer Function -func (oh *OtelHandler) InitOtelServer() error { - listenAddr := fmt.Sprintf("%s:%s", cfg.GlobalCfg.OtelGRPCListenAddr, cfg.GlobalCfg.OtelGRPCListenPort) - - // Start listening - lis, err := net.Listen("tcp", listenAddr) - if err != nil { - msg := fmt.Sprintf("unable to listen at %s: %v", listenAddr, err) - return errors.New(msg) - } - - // Create gRPC Server, register services - server := grpc.NewServer() - otelLogs.RegisterLogsServiceServer(server, olh) - - oh.listener = lis - oh.gRPCServer = server - - log.Printf("[OpenTelemetry] Server Listening at %s", listenAddr) - return nil -} - -// StartOtelServer Function -func (oh *OtelHandler) StartOtelServer(wg *sync.WaitGroup) error { - log.Printf("[OpenTelemetry] Starting server") - var err error - err = nil - - // Serve is blocking function - go func() { - wg.Add(1) - err = oh.gRPCServer.Serve(oh.listener) - if err != nil { - wg.Done() - return - } - - wg.Done() - }() - - return err -} - -// StopOtelServer Function -func (oh *OtelHandler) StopOtelServer() { - // Gracefully cleanup - oh.stopChan <- struct{}{} - - // Gracefully stop gRPC Server - oh.gRPCServer.GracefulStop() - - log.Printf("[OpenTelemetry] Stopped server") -} - -// OtelLogServer structure -type OtelLogServer struct { - otelLogs.UnimplementedLogsServiceServer -} - -// NewOtelLogServer Function -func NewOtelLogServer() *OtelLogServer { - return new(OtelLogServer) -} - -// Export Function -func (ols *OtelLogServer) Export(_ context.Context, req *otelLogs.ExportLogsServiceRequest) (*otelLogs.ExportLogsServiceResponse, error) { - // This is for Log.Export in OpenTelemetry format - als := GenerateAccessLogs(req.String()) - - for _, al := range als { - Lh.InsertLog(al) - } - - // For now, we will not consider partial success - ret := otelLogs.ExportLogsServiceResponse{ - PartialSuccess: nil, - } - - return &ret, nil -} diff --git a/sentryflow/core/sentryflow.go b/sentryflow/core/sentryflow.go index ce6b901..be6f5c6 100644 --- a/sentryflow/core/sentryflow.go +++ b/sentryflow/core/sentryflow.go @@ -18,62 +18,44 @@ func init() { StopChan = make(chan struct{}) } -// NumbatDaemon Structure -type NumbatDaemon struct { +// SentryFlowDaemon Structure +type SentryFlowDaemon struct { WgDaemon *sync.WaitGroup } -// NewNumbatDaemon Function -func NewNumbatDaemon() *NumbatDaemon { - dm := new(NumbatDaemon) +// NewSentryFlowDaemon Function +func NewSentryFlowDaemon() *SentryFlowDaemon { + dm := new(SentryFlowDaemon) dm.WgDaemon = new(sync.WaitGroup) return dm } -// DestroyNumbatDaemon Function -func (dm *NumbatDaemon) DestroyNumbatDaemon() { +// DestroySentryFlowDaemon Function +func (dm *SentryFlowDaemon) DestroySentryFlowDaemon() { } // watchK8s Function -func (dm *NumbatDaemon) watchK8s() { +func (dm *SentryFlowDaemon) watchK8s() { K8s.RunInformers(StopChan, dm.WgDaemon) } // logProcessor Function -func (dm *NumbatDaemon) logProcessor() { +func (dm *SentryFlowDaemon) logProcessor() { StartLogProcessor(dm.WgDaemon) log.Printf("[SentryFlow] Started log processor") } // metricAnalyzer Function -func (dm *NumbatDaemon) metricAnalyzer() { +func (dm *SentryFlowDaemon) metricAnalyzer() { metrics.StartMetricsAnalyzer(dm.WgDaemon) log.Printf("[SentryFlow] Started metric analyzer") } -// otelServer Function -func (dm *NumbatDaemon) otelServer() { - // Initialize and start OpenTelemetry Server - err := Oh.InitOtelServer() - if err != nil { - log.Fatalf("[SentryFlow] Unable to intialize OpenTelemetry Server: %v", err) - return - } - - err = Oh.StartOtelServer(dm.WgDaemon) - if err != nil { - log.Fatalf("[SentryFlow] Unable to start OpenTelemetry Server: %v", err) - return - } - - log.Printf("[SentryFlow] Started OpenTelemetry collector") -} - // exporterServer Function -func (dm *NumbatDaemon) exporterServer() { +func (dm *SentryFlowDaemon) exporterServer() { // Initialize and start exporter server err := exporter.Exp.InitExporterServer() if err != nil { @@ -89,7 +71,7 @@ func (dm *NumbatDaemon) exporterServer() { } // patchK8s Function -func (dm *NumbatDaemon) patchK8s() error { +func (dm *SentryFlowDaemon) patchK8s() error { err := K8s.PatchIstioConfigMap() if err != nil { return err @@ -115,12 +97,12 @@ func (dm *NumbatDaemon) patchK8s() error { // SentryFlow Function func SentryFlow() { // create a daemon - dm := NewNumbatDaemon() + dm := NewSentryFlowDaemon() // Initialize Kubernetes client if !K8s.InitK8sClient() { log.Printf("[Error] Failed to initialize Kubernetes client") - dm.DestroyNumbatDaemon() + dm.DestroySentryFlowDaemon() return } @@ -140,9 +122,6 @@ func SentryFlow() { // Start metric analyzer dm.metricAnalyzer() - // Start OpenTelemetry server - dm.otelServer() - // Start exporter server dm.exporterServer() diff --git a/sentryflow/exporter/exporterHandler.go b/sentryflow/exporter/exporterHandler.go index 10d43a6..5be91eb 100644 --- a/sentryflow/exporter/exporterHandler.go +++ b/sentryflow/exporter/exporterHandler.go @@ -27,13 +27,13 @@ func init() { type Handler struct { baseExecutionID uint64 currentLogCount uint64 - logChannel chan *protobuf.APILog - lock sync.Mutex // @todo find better solution for this stopChan chan struct{} - + lock sync.Mutex exporters []*Inform + metricExporters []*metricStreamInform exporterLock sync.Mutex exporterLogs chan *protobuf.APILog + exporterMetrics chan *protobuf.EnvoyMetric listener net.Listener gRPCServer *grpc.Server @@ -41,23 +41,30 @@ type Handler struct { // Inform structure type Inform struct { - stream protobuf.SentryFlow_GetLogServer + stream protobuf.SentryFlow_GetLogServer error chan error Hostname string IPAddress string } +type metricStreamInform struct { + metricStream protobuf.SentryFlow_GetEnvoyMetricsServer + error chan error + Hostname string + IPAddress string +} + // NewExporterHandler Function func NewExporterHandler() *Handler { exp := &Handler{ baseExecutionID: uint64(time.Now().UnixMicro()), currentLogCount: 0, exporters: make([]*Inform, 0), - logChannel: make(chan *protobuf.APILog), stopChan: make(chan struct{}), lock: sync.Mutex{}, exporterLock: sync.Mutex{}, exporterLogs: make(chan *protobuf.APILog), + exporterMetrics: make(chan *protobuf.EnvoyMetric), } return exp @@ -74,6 +81,11 @@ func InsertAccessLog(al *protobuf.APILog) { Exp.exporterLogs <- al } +//InsertEnvoyMetric Function +func InsertEnvoyMetric(em *protobuf.EnvoyMetric) { + Exp.exporterMetrics <- em +} + // InitExporterServer Function func (exp *Handler) InitExporterServer() error { listenAddr := fmt.Sprintf("%s:%s", cfg.GlobalCfg.CustomExportListenAddr, cfg.GlobalCfg.CustomExportListenPort) @@ -139,6 +151,17 @@ routineLoop: log.Printf("[Exporter] Log exporting failed %v:", err) } + case em, ok := <-exp.exporterMetrics: + if !ok { + log.Printf("[Exporter] EnvoyMetric exporter channel closed") + break routineLoop + } + + err := exp.sendMetrics(em) + if err != nil { + log.Printf("[Exporter] Metric exporting failed %v:", err) + } + case <-exp.stopChan: break routineLoop } @@ -188,6 +211,46 @@ func (exp *Handler) sendLogs(l *protobuf.APILog) error { return nil } +// sendMetrics Function +func (exp *Handler) sendMetrics(l *protobuf.EnvoyMetric) error { + exp.exporterLock.Lock() + defer exp.exporterLock.Unlock() + + // iterate and send logs + failed := 0 + total := len(exp.metricExporters) + for _, exporter := range exp.metricExporters { + curRetry := 0 + + // @todo: make max retry count per logs using config + // @todo: make max retry count per single exporter before removing the exporter using config + var err error + for curRetry < 3 { + err = exporter.metricStream.Send(l) + if err != nil { + log.Printf("[Exporter] Unable to send metric to %s(%s) retry=%d/%d: %v", + exporter.Hostname, exporter.IPAddress, curRetry, 3, err) + curRetry++ + } else { + break + } + } + + // Count failed + if err != nil { + failed++ + } + } + + // notify failed count + if failed != 0 { + msg := fmt.Sprintf("unable to send metrics properly %d/%d failed", failed, total) + return errors.New(msg) + } + + return nil +} + // StopExporterServer Function func (exp *Handler) StopExporterServer() { // Gracefully stop all client connections diff --git a/sentryflow/exporter/exporterServer.go b/sentryflow/exporter/exporterServer.go index 868648f..669701f 100644 --- a/sentryflow/exporter/exporterServer.go +++ b/sentryflow/exporter/exporterServer.go @@ -46,6 +46,25 @@ func (exs *Server) GetLog(info *protobuf.ClientInfo, stream protobuf.SentryFlow_ return <-curExporter.error } +func (exs *Server) GetEnvoyMetrics(info *protobuf.ClientInfo, stream protobuf.SentryFlow_GetEnvoyMetricsServer) error { + log.Printf("[Exporter] Client %s(%s) connected", info.HostName, info.IPAddress) + + curExporter := &metricStreamInform{ + metricStream: stream, + Hostname: info.HostName, + IPAddress: info.IPAddress, + } + + // Append new exporter client for future use + Exp.exporterLock.Lock() + Exp.metricExporters = append(Exp.metricExporters, curExporter) + Exp.exporterLock.Unlock() + + // Keeping gRPC stream alive + // refer https://stackoverflow.com/questions/36921131/ + return <-curExporter.error +} + // GetAPIMetrics Function func (exs *Server) GetAPIMetrics(_ context.Context, info *protobuf.ClientInfo) (*protobuf.APIMetric, error) { log.Printf("[Exporter] Client %s(%s) connected", info.HostName, info.IPAddress) diff --git a/sentryflow/go.mod b/sentryflow/go.mod index 45a2f35..a7e1f13 100644 --- a/sentryflow/go.mod +++ b/sentryflow/go.mod @@ -1,12 +1,13 @@ module github.com/5GSEC/sentryflow -go 1.21 +go 1.19 replace github.com/5GSEC/sentryflow/protobuf => ../protobuf require ( github.com/5GSEC/sentryflow/protobuf v0.0.0-00010101000000-000000000000 github.com/emicklei/go-restful/v3 v3.11.0 + github.com/envoyproxy/go-control-plane v0.11.1 github.com/spf13/viper v1.18.2 go.opentelemetry.io/proto/otlp v1.0.0 google.golang.org/grpc v1.61.1 @@ -17,14 +18,16 @@ require ( ) require ( + github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -40,6 +43,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -60,7 +64,7 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sentryflow/go.sum b/sentryflow/go.sum index 835e8a6..deab85b 100644 --- a/sentryflow/go.sum +++ b/sentryflow/go.sum @@ -1,3 +1,9 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -5,8 +11,13 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= @@ -18,17 +29,20 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -37,7 +51,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -52,7 +65,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -71,16 +83,15 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -119,11 +130,19 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -132,12 +151,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -160,6 +183,10 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -170,20 +197,26 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= -google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -197,6 +230,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= diff --git a/sentryflow/main.go b/sentryflow/main.go index a3b6381..b151946 100644 --- a/sentryflow/main.go +++ b/sentryflow/main.go @@ -3,18 +3,25 @@ package main import ( - cfg "github.com/5GSEC/sentryflow/config" - core "github.com/5GSEC/sentryflow/core" + "github.com/5GSEC/sentryflow/collector" + "github.com/5GSEC/sentryflow/core" _ "google.golang.org/grpc/encoding/gzip" // If not set, encoding problem occurs https://stackoverflow.com/questions/74062727 "log" ) // main is the entrypoint of this program func main() { - err := cfg.LoadConfig() + go func() { + core.SentryFlow() + }() + + err := collector.Ch.InitGRPCServer() if err != nil { - log.Fatalf("[SentryFlow] Unable to load config: %v", err) + log.Fatalf("[Error] Unable to start collector gRPC Server: %v", err) } - core.SentryFlow() + err = collector.Ch.Serve() + if err != nil { + log.Fatalf("[Error] Unable to serve gRPC Server: %v", err) + } } diff --git a/sentryflow/metrics/api/apiAnalyzer.go b/sentryflow/metrics/api/apiAnalyzer.go index 0798746..78a2ff7 100644 --- a/sentryflow/metrics/api/apiAnalyzer.go +++ b/sentryflow/metrics/api/apiAnalyzer.go @@ -19,6 +19,9 @@ type Analyzer struct { perAPICount map[string]uint64 perAPICountLock sync.Mutex // @todo perhaps combine those two? + curBatchCount int + batchCountLock sync.Mutex + stopChan chan struct{} apiJob chan string } diff --git a/sentryflow/types/metrics.go b/sentryflow/types/metrics.go new file mode 100644 index 0000000..f174e9d --- /dev/null +++ b/sentryflow/types/metrics.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +// PerAPICount Structure +type PerAPICount struct { + Api string + Count int +}