From 76a22898ff3f2124885cd022771462c5b140f4de Mon Sep 17 00:00:00 2001 From: Eray Arslan Date: Fri, 25 Oct 2024 11:40:36 +0300 Subject: [PATCH] feat: bump go-dcp to v1.2.0-rc.2 and tls improvements --- config/config.go | 2 ++ example/default-mapper/go.mod | 2 +- example/default-mapper/go.sum | 4 +-- example/grafana/go.mod | 2 +- example/grafana/go.sum | 4 +-- example/simple-logger/go.mod | 2 +- example/simple-logger/go.sum | 4 +-- example/simple/go.mod | 2 +- example/simple/go.sum | 4 +-- example/struct-config/go.mod | 2 +- example/struct-config/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- kafka/client.go | 51 ++++++++++++++++++++++++++--------- test/integration/go.mod | 2 +- test/integration/go.sum | 4 +-- 16 files changed, 62 insertions(+), 33 deletions(-) diff --git a/config/config.go b/config/config.go index 6d6e371..e118451 100644 --- a/config/config.go +++ b/config/config.go @@ -14,9 +14,11 @@ type Kafka struct { ProducerBatchBytes any `yaml:"producerBatchBytes"` CollectionTopicMapping map[string]string `yaml:"collectionTopicMapping"` InterCAPath string `yaml:"interCAPath"` + InterCA string `yaml:"interCA"` ScramUsername string `yaml:"scramUsername"` ScramPassword string `yaml:"scramPassword"` RootCAPath string `yaml:"rootCAPath"` + RootCA string `yaml:"rootCA"` ClientID string `yaml:"clientID"` Balancer string `yaml:"balancer"` Brokers []string `yaml:"brokers"` diff --git a/example/default-mapper/go.mod b/example/default-mapper/go.mod index 8e15c12..dad005a 100644 --- a/example/default-mapper/go.mod +++ b/example/default-mapper/go.mod @@ -7,7 +7,7 @@ replace github.com/Trendyol/go-dcp-kafka => ./../.. require github.com/Trendyol/go-dcp-kafka v0.0.0 require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb // indirect + github.com/Trendyol/go-dcp v1.2.0-rc.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect diff --git a/example/default-mapper/go.sum b/example/default-mapper/go.sum index 1b090be..f05e7c8 100644 --- a/example/default-mapper/go.sum +++ b/example/default-mapper/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/example/grafana/go.mod b/example/grafana/go.mod index 4de1fdb..e9e5390 100644 --- a/example/grafana/go.mod +++ b/example/grafana/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb // indirect + github.com/Trendyol/go-dcp v1.2.0-rc.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect diff --git a/example/grafana/go.sum b/example/grafana/go.sum index 501801f..5aeccc4 100644 --- a/example/grafana/go.sum +++ b/example/grafana/go.sum @@ -3,8 +3,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/example/simple-logger/go.mod b/example/simple-logger/go.mod index 294a981..d25510b 100644 --- a/example/simple-logger/go.mod +++ b/example/simple-logger/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb // indirect + github.com/Trendyol/go-dcp v1.2.0-rc.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect diff --git a/example/simple-logger/go.sum b/example/simple-logger/go.sum index 1b090be..f05e7c8 100644 --- a/example/simple-logger/go.sum +++ b/example/simple-logger/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/example/simple/go.mod b/example/simple/go.mod index 8e15c12..dad005a 100644 --- a/example/simple/go.mod +++ b/example/simple/go.mod @@ -7,7 +7,7 @@ replace github.com/Trendyol/go-dcp-kafka => ./../.. require github.com/Trendyol/go-dcp-kafka v0.0.0 require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb // indirect + github.com/Trendyol/go-dcp v1.2.0-rc.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/ansrivas/fiberprometheus/v2 v2.7.0 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect diff --git a/example/simple/go.sum b/example/simple/go.sum index 1b090be..f05e7c8 100644 --- a/example/simple/go.sum +++ b/example/simple/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/example/struct-config/go.mod b/example/struct-config/go.mod index 9c04c81..e14d50e 100644 --- a/example/struct-config/go.mod +++ b/example/struct-config/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/Trendyol/go-dcp-kafka => ./../.. require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb + github.com/Trendyol/go-dcp v1.2.0-rc.2 github.com/Trendyol/go-dcp-kafka v0.0.0 ) diff --git a/example/struct-config/go.sum b/example/struct-config/go.sum index 1b090be..f05e7c8 100644 --- a/example/struct-config/go.sum +++ b/example/struct-config/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/go.mod b/go.mod index 903ccab..2b2dbb9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Trendyol/go-dcp-kafka go 1.20 require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb + github.com/Trendyol/go-dcp v1.2.0-rc.2 github.com/json-iterator/go v1.1.12 github.com/prometheus/client_golang v1.20.5 github.com/segmentio/kafka-go v0.4.47 diff --git a/go.sum b/go.sum index 1b090be..f05e7c8 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg= diff --git a/kafka/client.go b/kafka/client.go index 2628313..69903b3 100644 --- a/kafka/client.go +++ b/kafka/client.go @@ -46,28 +46,53 @@ func newTLSContent( scramUsername, scramPassword, rootCAPath, - interCAPath string, + interCAPath, + rootCA, + interCA string, ) (*tlsContent, error) { mechanism, err := scram.Mechanism(scram.SHA512, scramUsername, scramPassword) if err != nil { return nil, err } - caCert, err := os.ReadFile(os.ExpandEnv(rootCAPath)) - if err != nil { - logger.Log.Error("an error occurred while reading ca.pem file! Error: %s", err.Error()) - return nil, err + certCount := 0 + caCertPool := x509.NewCertPool() + + if rootCAPath != "" { + caCert, err := os.ReadFile(os.ExpandEnv(rootCAPath)) + if err != nil { + logger.Log.Error("an error occurred while reading ca.pem file! Error: %s", err.Error()) + return nil, err + } + caCertPool.AppendCertsFromPEM(caCert) + certCount++ } - intCert, err := os.ReadFile(os.ExpandEnv(interCAPath)) - if err != nil { - logger.Log.Error("an error occurred while reading int.pem file! Error: %s", err.Error()) - return nil, err + if interCAPath != "" { + intCert, err := os.ReadFile(os.ExpandEnv(interCAPath)) + if err != nil { + logger.Log.Error("an error occurred while reading int.pem file! Error: %s", err.Error()) + return nil, err + } + caCertPool.AppendCertsFromPEM(intCert) + certCount++ } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - caCertPool.AppendCertsFromPEM(intCert) + if rootCA != "" { + caCertPool.AppendCertsFromPEM([]byte(rootCA)) + certCount++ + } + + if interCA != "" { + caCertPool.AppendCertsFromPEM([]byte(interCA)) + certCount++ + } + + if certCount == 0 { + err := errors.New("certPool is empty") + logger.Log.Error("an error occurred while creating tls content! Error: %s", err.Error()) + return nil, err + } return &tlsContent{ config: &tls.Config{ @@ -271,6 +296,8 @@ func NewClient(config *config.Connector) Client { config.Kafka.ScramPassword, config.Kafka.RootCAPath, config.Kafka.InterCAPath, + config.Kafka.RootCA, + config.Kafka.InterCA, ) if err != nil { panic(err) diff --git a/test/integration/go.mod b/test/integration/go.mod index 226233c..23e4700 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/Trendyol/go-dcp-kafka => ../../. require ( - github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb + github.com/Trendyol/go-dcp v1.2.0-rc.2 github.com/Trendyol/go-dcp-kafka v0.0.0 github.com/segmentio/kafka-go v0.4.47 ) diff --git a/test/integration/go.sum b/test/integration/go.sum index 1b090be..f05e7c8 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -1,8 +1,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb h1:WTud3h2kkzFhIZP1aIo+b2q8FonAjmgi8wZ0lT63Cs4= -github.com/Trendyol/go-dcp v1.1.57-0.20241023075515-0762009134fb/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= +github.com/Trendyol/go-dcp v1.2.0-rc.2 h1:7a++gznhbrTKqv8JEx3tCAKhUGD1WIPqeXhPE6Duc/4= +github.com/Trendyol/go-dcp v1.2.0-rc.2/go.mod h1:k49rj06gaLt3lLNoWdQFT+KzXb0C5mzaByRwEgeTu20= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/ansrivas/fiberprometheus/v2 v2.7.0 h1:09XiSzG0J7aZp7RviklngdWdDbSybKjhuWAstp003Gg=