diff --git a/config/samples/cluster_v1_local.yaml b/config/samples/cluster_v1_local.yaml index a69ddc62..d5865df8 100644 --- a/config/samples/cluster_v1_local.yaml +++ b/config/samples/cluster_v1_local.yaml @@ -3,7 +3,7 @@ kind: Ytsaurus metadata: name: minisaurus spec: - coreImage: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-06-ee9e61d820c81624e2562edca351e9bc8bdd7e2e + coreImage: ghcr.io/ytsaurus/ytsaurus:stable-23.2.0-relwithdebinfo discovery: instanceCount: 1 @@ -28,12 +28,6 @@ spec: minLogLevel: error writerType: stderr - kafkaProxies: - - serviceType: NodePort - loggers: *loggers - instanceCount: 1 - role: default - primaryMasters: instanceCount: 1 cellTag: 1 diff --git a/config/samples/cluster_v1_local_with_kafka.yaml b/config/samples/cluster_v1_local_with_kafka.yaml new file mode 100644 index 00000000..bce2edb4 --- /dev/null +++ b/config/samples/cluster_v1_local_with_kafka.yaml @@ -0,0 +1,171 @@ +apiVersion: cluster.ytsaurus.tech/v1 +kind: Ytsaurus +metadata: + name: minisaurus +spec: + coreImage: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-06-ee9e61d820c81624e2562edca351e9bc8bdd7e2e + + discovery: + instanceCount: 1 + + # Make reusable loggers config with yaml anchor. + loggers: &loggers + - name: debug + compression: zstd + minLogLevel: debug + writerType: file + rotationPolicy: &rotationPolicy + maxTotalSizeToKeep: 10000000 + rotationPeriodMilliseconds: 900000 + categoriesFilter: + type: exclude + values: [ "Bus", "Concurrency" ] + - name: info + minLogLevel: info + writerType: file + rotationPolicy: *rotationPolicy + - name: error + minLogLevel: error + writerType: stderr + + kafkaProxies: + - serviceType: NodePort + loggers: *loggers + instanceCount: 1 + role: default + image: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-26-d79040f9b4037aabf1f6b686e442cc029679a285 + + primaryMasters: + instanceCount: 1 + cellTag: 1 + loggers: *loggers + locations: + - locationType: MasterChangelogs + path: /yt/master-data/master-changelogs + - locationType: MasterSnapshots + path: /yt/master-data/master-snapshots + + volumeMounts: + - name: master-data + mountPath: /yt/master-data + + volumeClaimTemplates: + - metadata: + name: master-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 5Gi + + httpProxies: + - serviceType: NodePort + loggers: *loggers + instanceCount: 1 + role: default + - serviceType: NodePort + loggers: *loggers + instanceCount: 1 + role: control + + rpcProxies: + - instanceCount: 1 + loggers: *loggers + role: default + - instanceCount: 1 + loggers: *loggers + role: heavy + + dataNodes: + - instanceCount: 3 + loggers: *loggers + + volumeMounts: + - name: node-data + mountPath: /yt/node-data + + locations: + - locationType: ChunkStore + path: /yt/node-data/chunk-store + + volumeClaimTemplates: + - metadata: + name: node-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 5Gi + + execNodes: + - instanceCount: 1 + loggers: *loggers + resources: + limits: + cpu: 1 + memory: 2Gi + + volumeMounts: + - name: node-data + mountPath: /yt/node-data + + volumes: + - name: node-data + emptyDir: + sizeLimit: 5Gi + + locations: + - locationType: ChunkCache + path: /yt/node-data/chunk-cache + - locationType: Slots + path: /yt/node-data/slots + + jobProxyLoggers: + - name: debug + compression: zstd + minLogLevel: debug + writerType: file + useTimestampSuffix: true + rotationPolicy: &rotationPolicy + maxTotalSizeToKeep: 10000000 + rotationPeriodMilliseconds: 900000 + categoriesFilter: + type: exclude + values: [ "Bus", "Concurrency" ] + - name: info + minLogLevel: info + writerType: file + rotationPolicy: *rotationPolicy + - name: error + minLogLevel: error + writerType: stderr + + schedulers: + instanceCount: 1 + loggers: *loggers + + controllerAgents: + instanceCount: 1 + loggers: *loggers + + ui: + image: ghcr.io/ytsaurus/ui:stable + serviceType: NodePort + instanceCount: 1 + + strawberry: + resources: + limits: + memory: 100Mi + image: ghcr.io/ytsaurus/strawberry:0.0.11 + + tabletNodes: + - instanceCount: 1 + + yqlAgents: + instanceCount: 1 + image: ghcr.io/ytsaurus/query-tracker:0.0.6 + + queryTrackers: + instanceCount: 1 + image: ghcr.io/ytsaurus/query-tracker:0.0.6 diff --git a/pkg/components/kafka_proxy.go b/pkg/components/kafka_proxy.go index 5456c28c..505dbafa 100644 --- a/pkg/components/kafka_proxy.go +++ b/pkg/components/kafka_proxy.go @@ -41,6 +41,16 @@ func NewKafkaProxy(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, maste func() ([]byte, error) { return cfgen.GetKafkaProxyConfig(spec) }, + WithContainerPorts(corev1.ContainerPort{ + Name: consts.YTRPCPortName, + ContainerPort: consts.KafkaProxyRPCPort, + Protocol: corev1.ProtocolTCP, + }), + WithContainerPorts(corev1.ContainerPort{ + Name: consts.KafkaPortName, + ContainerPort: consts.KafkaProxyKafkaPort, + Protocol: corev1.ProtocolTCP, + }), ) var balancingService *resources.RPCService = nil diff --git a/pkg/consts/address.go b/pkg/consts/address.go index 689bc4d3..ef33f9d0 100644 --- a/pkg/consts/address.go +++ b/pkg/consts/address.go @@ -2,6 +2,7 @@ package consts const ( YTRPCPortName = "rpc" + KafkaPortName = "kafka" YTMonitoringContainerPortName = "metrics" YTMonitoringServicePortName = "ytsaurus-metrics" @@ -62,5 +63,6 @@ const ( MasterCachesMonitoringPort = 10018 KafkaProxyMonitoringPort = 10020 - KafkaProxyKafkaPort = 80 + KafkaProxyKafkaPort = 9023 + KafkaProxyRPCPort = 9021 ) diff --git a/pkg/consts/labels.go b/pkg/consts/labels.go index db9a5cba..ae2c16e1 100644 --- a/pkg/consts/labels.go +++ b/pkg/consts/labels.go @@ -35,7 +35,7 @@ func ComponentLabel(component ComponentType) string { case TcpProxyType: return "yt-tcp-proxy" case KafkaProxyType: - return "yt-kp-proxy" + return "yt-kafka-proxy" case QueueAgentType: return "yt-queue-agent" case QueryTrackerType: diff --git a/pkg/ytconfig/proxy.go b/pkg/ytconfig/proxy.go index 9010e807..11539d8a 100644 --- a/pkg/ytconfig/proxy.go +++ b/pkg/ytconfig/proxy.go @@ -198,6 +198,7 @@ func getKafkaProxyServerCarcass(spec *ytv1.KafkaProxiesSpec) (KafkaProxyServer, var c KafkaProxyServer c.MonitoringPort = *spec.InstanceSpec.MonitoringPort + c.RPCPort = consts.KafkaProxyRPCPort c.Role = spec.Role c.Auth.RequireAuthentication = true