Fixes

ytsaurus · Dec 26, 2024 · 89a8341 · 89a8341
1 parent f876a15
commit 89a8341
Show file tree

Hide file tree

Showing 6 changed files with 187 additions and 9 deletions.
diff --git a/config/samples/cluster_v1_local.yaml b/config/samples/cluster_v1_local.yaml
@@ -3,7 +3,7 @@ kind: Ytsaurus
 metadata:
   name: minisaurus
 spec:
-  coreImage: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-06-ee9e61d820c81624e2562edca351e9bc8bdd7e2e
+  coreImage: ghcr.io/ytsaurus/ytsaurus:stable-23.2.0-relwithdebinfo
 
   discovery:
     instanceCount: 1
@@ -28,12 +28,6 @@ spec:
         minLogLevel: error
         writerType: stderr
 
-  kafkaProxies:
-    - serviceType: NodePort
-      loggers: *loggers
-      instanceCount: 1
-      role: default
-
   primaryMasters:
     instanceCount: 1
     cellTag: 1

diff --git a/config/samples/cluster_v1_local_with_kafka.yaml b/config/samples/cluster_v1_local_with_kafka.yaml
@@ -0,0 +1,171 @@
+apiVersion: cluster.ytsaurus.tech/v1
+kind: Ytsaurus
+metadata:
+  name: minisaurus
+spec:
+  coreImage: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-06-ee9e61d820c81624e2562edca351e9bc8bdd7e2e
+
+  discovery:
+    instanceCount: 1
+
+    # Make reusable loggers config with yaml anchor.
+    loggers: &loggers
+      - name: debug
+        compression: zstd
+        minLogLevel: debug
+        writerType: file
+        rotationPolicy: &rotationPolicy
+          maxTotalSizeToKeep: 10000000
+          rotationPeriodMilliseconds: 900000
+        categoriesFilter:
+          type: exclude
+          values: [ "Bus", "Concurrency" ]
+      - name: info
+        minLogLevel: info
+        writerType: file
+        rotationPolicy: *rotationPolicy
+      - name: error
+        minLogLevel: error
+        writerType: stderr
+
+  kafkaProxies:
+    - serviceType: NodePort
+      loggers: *loggers
+      instanceCount: 1
+      role: default
+      image: ghcr.io/ytsaurus/ytsaurus-nightly:dev-24.2-2024-12-26-d79040f9b4037aabf1f6b686e442cc029679a285
+
+  primaryMasters:
+    instanceCount: 1
+    cellTag: 1
+    loggers: *loggers
+    locations:
+      - locationType: MasterChangelogs
+        path: /yt/master-data/master-changelogs
+      - locationType: MasterSnapshots
+        path: /yt/master-data/master-snapshots
+
+    volumeMounts:
+      - name: master-data
+        mountPath: /yt/master-data
+
+    volumeClaimTemplates:
+      - metadata:
+          name: master-data
+        spec:
+          accessModes: [ "ReadWriteOnce" ]
+          resources:
+            requests:
+              storage: 5Gi
+
+  httpProxies:
+    - serviceType: NodePort
+      loggers: *loggers
+      instanceCount: 1
+      role: default
+    - serviceType: NodePort
+      loggers: *loggers
+      instanceCount: 1
+      role: control
+
+  rpcProxies:
+    - instanceCount: 1
+      loggers: *loggers
+      role: default
+    - instanceCount: 1
+      loggers: *loggers
+      role: heavy
+
+  dataNodes:
+    - instanceCount: 3
+      loggers: *loggers
+
+      volumeMounts:
+        - name: node-data
+          mountPath: /yt/node-data
+
+      locations:
+        - locationType: ChunkStore
+          path: /yt/node-data/chunk-store
+
+      volumeClaimTemplates:
+        - metadata:
+            name: node-data
+          spec:
+            accessModes: [ "ReadWriteOnce" ]
+            resources:
+              requests:
+                storage: 5Gi
+
+  execNodes:
+    - instanceCount: 1
+      loggers: *loggers
+      resources:
+        limits:
+          cpu: 1
+          memory: 2Gi
+
+      volumeMounts:
+        - name: node-data
+          mountPath: /yt/node-data
+
+      volumes:
+        - name: node-data
+          emptyDir:
+            sizeLimit: 5Gi
+
+      locations:
+        - locationType: ChunkCache
+          path: /yt/node-data/chunk-cache
+        - locationType: Slots
+          path: /yt/node-data/slots
+
+      jobProxyLoggers:
+        - name: debug
+          compression: zstd
+          minLogLevel: debug
+          writerType: file
+          useTimestampSuffix: true
+          rotationPolicy: &rotationPolicy
+            maxTotalSizeToKeep: 10000000
+            rotationPeriodMilliseconds: 900000
+          categoriesFilter:
+            type: exclude
+            values: [ "Bus", "Concurrency" ]
+        - name: info
+          minLogLevel: info
+          writerType: file
+          rotationPolicy: *rotationPolicy
+        - name: error
+          minLogLevel: error
+          writerType: stderr
+
+  schedulers:
+    instanceCount: 1
+    loggers: *loggers
+
+  controllerAgents:
+    instanceCount: 1
+    loggers: *loggers
+
+  ui:
+    image: ghcr.io/ytsaurus/ui:stable
+    serviceType: NodePort
+    instanceCount: 1
+
+  strawberry:
+    resources:
+      limits:
+        memory: 100Mi
+    image: ghcr.io/ytsaurus/strawberry:0.0.11
+
+  tabletNodes:
+    - instanceCount: 1
+
+  yqlAgents:
+    instanceCount: 1
+    image: ghcr.io/ytsaurus/query-tracker:0.0.6
+
+  queryTrackers:
+    instanceCount: 1
+    image: ghcr.io/ytsaurus/query-tracker:0.0.6
diff --git a/pkg/components/kafka_proxy.go b/pkg/components/kafka_proxy.go
@@ -41,6 +41,16 @@ func NewKafkaProxy(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, maste
 		func() ([]byte, error) {
 			return cfgen.GetKafkaProxyConfig(spec)
 		},
+		WithContainerPorts(corev1.ContainerPort{
+			Name:          consts.YTRPCPortName,
+			ContainerPort: consts.KafkaProxyRPCPort,
+			Protocol:      corev1.ProtocolTCP,
+		}),
+		WithContainerPorts(corev1.ContainerPort{
+			Name:          consts.KafkaPortName,
+			ContainerPort: consts.KafkaProxyKafkaPort,
+			Protocol:      corev1.ProtocolTCP,
+		}),
 	)
 
 	var balancingService *resources.RPCService = nil

diff --git a/pkg/consts/address.go b/pkg/consts/address.go
@@ -2,6 +2,7 @@ package consts
 
 const (
 	YTRPCPortName = "rpc"
+	KafkaPortName = "kafka"
 
 	YTMonitoringContainerPortName = "metrics"
 	YTMonitoringServicePortName   = "ytsaurus-metrics"
@@ -62,5 +63,6 @@ const (
 	MasterCachesMonitoringPort = 10018
 
 	KafkaProxyMonitoringPort = 10020
-	KafkaProxyKafkaPort      = 80
+	KafkaProxyKafkaPort      = 9023
+	KafkaProxyRPCPort        = 9021
 )
diff --git a/pkg/consts/labels.go b/pkg/consts/labels.go
@@ -35,7 +35,7 @@ func ComponentLabel(component ComponentType) string {
 	case TcpProxyType:
 		return "yt-tcp-proxy"
 	case KafkaProxyType:
-		return "yt-kp-proxy"
+		return "yt-kafka-proxy"
 	case QueueAgentType:
 		return "yt-queue-agent"
 	case QueryTrackerType:

diff --git a/pkg/ytconfig/proxy.go b/pkg/ytconfig/proxy.go
@@ -198,6 +198,7 @@ func getKafkaProxyServerCarcass(spec *ytv1.KafkaProxiesSpec) (KafkaProxyServer,
 	var c KafkaProxyServer
 
 	c.MonitoringPort = *spec.InstanceSpec.MonitoringPort
+	c.RPCPort = consts.KafkaProxyRPCPort
 
 	c.Role = spec.Role
 	c.Auth.RequireAuthentication = true