Skip to content

Commit

Permalink
Merge pull request #49 from ydb-platform/topic-control-plane-flags
Browse files Browse the repository at this point in the history 
sync ydb topic protos to ydb
  • Loading branch information
@rekby
rekby authored Oct 30, 2024
2 parents 171be88 + 3b054ea commit 3744873
Showing 1 changed file with 130 additions and 15 deletions.
145 changes: 130 additions & 15 deletions protos/ydb_topic.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ message StreamReadMessage {
StopPartitionSessionRequest stop_partition_session_request = 9;

UpdatePartitionSession update_partition_session = 10;
EndPartitionSession end_partition_session = 11;
}
}

Expand All @@ -326,6 +327,8 @@ message StreamReadMessage {
string reader_name = 3;
// Direct reading from a partition node.
bool direct_read = 4;
// Indicates that the SDK supports auto partitioning.
bool auto_partitioning_support = 5;

message TopicReadSettings {
// Topic path.
Expand Down Expand Up @@ -491,7 +494,7 @@ message StreamReadMessage {
PartitionLocation partition_location = 4;
}

// Signal for server that cient is ready to recive data for partition.
// Signal for server that client is ready to recive data for partition.
message StartPartitionSessionResponse {
// Partition session identifier of partition to start read.
int64 partition_session_id = 1;
Expand Down Expand Up @@ -564,24 +567,41 @@ message StreamReadMessage {
int64 direct_read_id = 2;
}

// Signal from server that client has finished reading the partition and all messages have been read.
// Once a partition has been finished no further messages will ever arrive to that partition.
// This command is a hint to the client to commit offsets, after which the child partitions will be balanced independently in different reading sessions.
// Unlike StopPartitionSessionRequest, the client does not have to close the reading session.
// Client should not send a response to the command.
message EndPartitionSession {
// Partition session identifier.
int64 partition_session_id = 1;

// Ids of partitions which were merged with the ended partition.
repeated int64 adjacent_partition_ids = 2;
// Ids of partitions which was formed when the ended partition was split or merged.
repeated int64 child_partition_ids = 3;
}
}

// Messages for bidirectional streaming rpc StreamDirectRead
message StreamDirectReadMessage {

// Client-server message for direct read session.
// InitDirectRead - command from client to create and start a direct read session.
// StartDirectReadPartitionSession - command from client to create and start a direct read partition session.
// InitRequest - command from client to create and start a direct read session.
// StartDirectReadPartitionSessionRequest - command from client to create and start a direct read partition session.
// Client signals it is ready to get data from partition.
// UpdateTokenRequest - request to update auth token
message FromClient {
oneof client_message {
InitDirectRead init_direct_read = 1;
StartDirectReadPartitionSession start_direct_read_partition_session = 2;
InitRequest init_request = 1;
StartDirectReadPartitionSessionRequest start_direct_read_partition_session_request = 2;
UpdateTokenRequest update_token_request = 3;
}
}

// Server-client message for direct read session.
// InitResponse - correct handshake response.
// StartDirectReadPartitionSessionResponse - Response to StartDirectReadPartitionSessionRequest.
// DirectReadResponse - portion of message data.
// StopDirectReadPartitionSession - command from server to stop a direct read partition session.
// UpdateTokenResponse - acknowledgment of token update.
Expand All @@ -592,8 +612,9 @@ message StreamDirectReadMessage {
// Issues if any.
repeated Ydb.Issue.IssueMessage issues = 2;


oneof server_message {
InitResponse init_response = 6;
StartDirectReadPartitionSessionResponse start_direct_read_partition_session_response = 7;
StopDirectReadPartitionSession stop_direct_read_partition_session = 3;
DirectReadResponse direct_read_response = 4;
UpdateTokenResponse update_token_response = 5;
Expand All @@ -602,7 +623,7 @@ message StreamDirectReadMessage {

// Command from client to create and start a direct read session.
// Server should not send a response to the command.
message InitDirectRead {
message InitRequest {
// Read session identifier.
string session_id = 1;
// Topics that will be read by this session.
Expand All @@ -616,9 +637,12 @@ message StreamDirectReadMessage {
}
}

// Response to the handshake.
message InitResponse {
}

// Command from client to create and start a direct read partition session.
// Server should not send a response to the command.
message StartDirectReadPartitionSession {
message StartDirectReadPartitionSessionRequest {
// Partition session identifier.
int64 partition_session_id = 1;

Expand All @@ -629,6 +653,15 @@ message StreamDirectReadMessage {
int64 generation = 3;
}

// Signal for server that client is ready to receive data for partition.
message StartDirectReadPartitionSessionResponse {
// Partition session identifier of partition to start read.
int64 partition_session_id = 1;

// Partition generation.
int64 generation = 2;
}

// Command from server to stop a direct read partition session.
// Client should not send a response to the command.
message StopDirectReadPartitionSession {
Expand All @@ -640,8 +673,10 @@ message StreamDirectReadMessage {

// Partition session identifier.
int64 partition_session_id = 3;
}

// Partition generation.
int64 generation = 4;
}

// Messages that have been read directly from the partition node.
// It's a response to StreamRead.ReadRequest
Expand Down Expand Up @@ -802,24 +837,95 @@ message AlterConsumer {
map<string, string> alter_attributes = 6;
}

enum AutoPartitioningStrategy {
// The auto partitioning algorithm is not specified. The default value will be used.
AUTO_PARTITIONING_STRATEGY_UNSPECIFIED = 0;
// The auto partitioning is disabled.
AUTO_PARTITIONING_STRATEGY_DISABLED = 1;
// The auto partitioning algorithm will increase partitions count depending on the load characteristics.
// The auto partitioning algorithm will never decrease the number of partitions.
AUTO_PARTITIONING_STRATEGY_SCALE_UP = 2;
// The auto partitioning algorithm will both increase and decrease partitions count depending on the load characteristics.
AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN = 3;
// The auto partitioning is paused.
AUTO_PARTITIONING_STRATEGY_PAUSED = 4;
}

// Partitioning settings for topic.
message PartitioningSettings {
// Minimum partition count auto merge would stop working at.
// Auto merge would stop working when the partitions count reaches min_active_partitions
// Zero value means default - 1.
int64 min_active_partitions = 1 [(Ydb.value) = ">= 0"];
// Auto split would stop working when the partitions count reaches max_active_partitions
// Zero value means default - 1.
int64 max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
int64 partition_count_limit = 2 [(Ydb.value) = ">= 0"];
// Use max_active_partitions
int64 partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
// Settings for the partitions count auto partitioning.
AutoPartitioningSettings auto_partitioning_settings = 4;
}

message AutoPartitioningSettings {
// Strategy of auto partitioning.
AutoPartitioningStrategy strategy = 1;
// Partition write speed auto partitioning options.
AutoPartitioningWriteSpeedStrategy partition_write_speed = 2;
}

message AutoPartitioningWriteSpeedStrategy {
//Partition will be auto partitioned up (divided into 2 partitions)
//after write speed to the partition exceeds up_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window

//Partition will become a candidate to the auto partitioned down
//after write speed doesn’t reach down_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
//This candidate partition will be auto partitioned down when other neighbour partition will become a candidate to the auto partitioning down and not earlier than a retention period.

// Zero value means default - 300.
google.protobuf.Duration stabilization_window = 1;
// Zero value means default - 90.
int32 up_utilization_percent = 2 [(Ydb.value) = ">= 0"];
// Zero value means default - 30.
int32 down_utilization_percent = 3 [(Ydb.value) = ">= 0"];
}

// Partitioning settings for topic.
message AlterPartitioningSettings {
// Minimum partition count auto merge would stop working at.
// Zero value means default - 1.
optional int64 set_min_active_partitions = 1 [(Ydb.value) = ">= 0"];
// Maximum partition count auto merge would stop working at.
// Zero value means default - 1.
optional int64 set_max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0"];
// Use set_max_active_partitions
optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
// Settings for auto partitioning the partition number
optional AlterAutoPartitioningSettings alter_auto_partitioning_settings = 4;
}

message AlterAutoPartitioningSettings {
// Strategy of auto partitioning
optional AutoPartitioningStrategy set_strategy = 1;
// Auto partitioning write speed options.
optional AlterAutoPartitioningWriteSpeedStrategy set_partition_write_speed = 2;
}

message AlterAutoPartitioningWriteSpeedStrategy {
// The time of exceeding the threshold value, after which the partition will be
// auto partitioning.
// Zero value means default - 300.
optional google.protobuf.Duration set_stabilization_window = 1;
// The threshold value of the write speed to the partition as a percentage, when exceeded,
// the partition will be auto split.
// Zero value means default - 90.
optional int32 set_up_utilization_percent = 2 [(Ydb.value) = ">= 0"];
// The threshold value of the write speed to the partition as a percentage, if it is not reached,
// the partition will be auto merged.
// Zero value means default - 30.
optional int32 set_down_utilization_percent = 3 [(Ydb.value) = ">= 0"];
}

// Metering mode specifies the method used to determine consumption of resources by the topic.
Expand Down Expand Up @@ -920,6 +1026,13 @@ message DescribeTopicResponse {
Ydb.Operations.Operation operation = 1;
}

message PartitionKeyRange {
// Inclusive left border. Emptiness means -inf.
optional bytes from_bound = 1;
// Exclusive right border. Emptiness means +inf.
optional bytes to_bound = 2;
}

// Describe topic result message that will be inside DescribeTopicResponse.operation.
message DescribeTopicResult {
// Description of scheme object.
Expand Down Expand Up @@ -988,6 +1101,8 @@ message DescribeTopicResult {

// Partition location, filled only when include_location in request is true.
PartitionLocation partition_location = 6;

PartitionKeyRange key_range = 7;
}

message TopicStats {
Expand Down Expand Up @@ -1027,7 +1142,7 @@ message DescribePartitionResponse {
Ydb.Operations.Operation operation = 1;
}

// Describe partition result message that will be inside DescribeTopicResponse.operation.
// Describe partition result message that will be inside DescribePartitionResponse.operation.
message DescribePartitionResult {
// Partitions description.
DescribeTopicResult.PartitionInfo partition = 1;
Expand Down Expand Up @@ -1200,4 +1315,4 @@ message DropTopicResponse {

// Drop topic result message that will be inside DropTopicResponse.operation.
message DropTopicResult {
}
}

0 comments on commit 3744873

Please sign in to comment.