Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sync ydb topic protos to ydb #49

Merged
merged 1 commit into from
Oct 30, 2024
Merged

sync ydb topic protos to ydb #49

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 130 additions & 15 deletions protos/ydb_topic.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ message StreamReadMessage {
StopPartitionSessionRequest stop_partition_session_request = 9;

UpdatePartitionSession update_partition_session = 10;
EndPartitionSession end_partition_session = 11;
}
}

Expand All @@ -326,6 +327,8 @@ message StreamReadMessage {
string reader_name = 3;
// Direct reading from a partition node.
bool direct_read = 4;
// Indicates that the SDK supports auto partitioning.
bool auto_partitioning_support = 5;

message TopicReadSettings {
// Topic path.
Expand Down Expand Up @@ -491,7 +494,7 @@ message StreamReadMessage {
PartitionLocation partition_location = 4;
}

// Signal for server that cient is ready to recive data for partition.
// Signal for server that client is ready to recive data for partition.
message StartPartitionSessionResponse {
// Partition session identifier of partition to start read.
int64 partition_session_id = 1;
Expand Down Expand Up @@ -564,24 +567,41 @@ message StreamReadMessage {
int64 direct_read_id = 2;
}

// Signal from server that client has finished reading the partition and all messages have been read.
// Once a partition has been finished no further messages will ever arrive to that partition.
// This command is a hint to the client to commit offsets, after which the child partitions will be balanced independently in different reading sessions.
// Unlike StopPartitionSessionRequest, the client does not have to close the reading session.
// Client should not send a response to the command.
message EndPartitionSession {
// Partition session identifier.
int64 partition_session_id = 1;

// Ids of partitions which were merged with the ended partition.
repeated int64 adjacent_partition_ids = 2;
// Ids of partitions which was formed when the ended partition was split or merged.
repeated int64 child_partition_ids = 3;
}
}

// Messages for bidirectional streaming rpc StreamDirectRead
message StreamDirectReadMessage {

// Client-server message for direct read session.
// InitDirectRead - command from client to create and start a direct read session.
// StartDirectReadPartitionSession - command from client to create and start a direct read partition session.
// InitRequest - command from client to create and start a direct read session.
// StartDirectReadPartitionSessionRequest - command from client to create and start a direct read partition session.
// Client signals it is ready to get data from partition.
// UpdateTokenRequest - request to update auth token
message FromClient {
oneof client_message {
InitDirectRead init_direct_read = 1;
StartDirectReadPartitionSession start_direct_read_partition_session = 2;
InitRequest init_request = 1;
StartDirectReadPartitionSessionRequest start_direct_read_partition_session_request = 2;
UpdateTokenRequest update_token_request = 3;
}
}

// Server-client message for direct read session.
// InitResponse - correct handshake response.
// StartDirectReadPartitionSessionResponse - Response to StartDirectReadPartitionSessionRequest.
// DirectReadResponse - portion of message data.
// StopDirectReadPartitionSession - command from server to stop a direct read partition session.
// UpdateTokenResponse - acknowledgment of token update.
Expand All @@ -592,8 +612,9 @@ message StreamDirectReadMessage {
// Issues if any.
repeated Ydb.Issue.IssueMessage issues = 2;


oneof server_message {
InitResponse init_response = 6;
StartDirectReadPartitionSessionResponse start_direct_read_partition_session_response = 7;
StopDirectReadPartitionSession stop_direct_read_partition_session = 3;
DirectReadResponse direct_read_response = 4;
UpdateTokenResponse update_token_response = 5;
Expand All @@ -602,7 +623,7 @@ message StreamDirectReadMessage {

// Command from client to create and start a direct read session.
// Server should not send a response to the command.
message InitDirectRead {
message InitRequest {
// Read session identifier.
string session_id = 1;
// Topics that will be read by this session.
Expand All @@ -616,9 +637,12 @@ message StreamDirectReadMessage {
}
}

// Response to the handshake.
message InitResponse {
}

// Command from client to create and start a direct read partition session.
// Server should not send a response to the command.
message StartDirectReadPartitionSession {
message StartDirectReadPartitionSessionRequest {
// Partition session identifier.
int64 partition_session_id = 1;

Expand All @@ -629,6 +653,15 @@ message StreamDirectReadMessage {
int64 generation = 3;
}

// Signal for server that client is ready to receive data for partition.
message StartDirectReadPartitionSessionResponse {
// Partition session identifier of partition to start read.
int64 partition_session_id = 1;

// Partition generation.
int64 generation = 2;
}

// Command from server to stop a direct read partition session.
// Client should not send a response to the command.
message StopDirectReadPartitionSession {
Expand All @@ -640,8 +673,10 @@ message StreamDirectReadMessage {

// Partition session identifier.
int64 partition_session_id = 3;
}

// Partition generation.
int64 generation = 4;
}

// Messages that have been read directly from the partition node.
// It's a response to StreamRead.ReadRequest
Expand Down Expand Up @@ -802,24 +837,95 @@ message AlterConsumer {
map<string, string> alter_attributes = 6;
}

enum AutoPartitioningStrategy {
// The auto partitioning algorithm is not specified. The default value will be used.
AUTO_PARTITIONING_STRATEGY_UNSPECIFIED = 0;
// The auto partitioning is disabled.
AUTO_PARTITIONING_STRATEGY_DISABLED = 1;
// The auto partitioning algorithm will increase partitions count depending on the load characteristics.
// The auto partitioning algorithm will never decrease the number of partitions.
AUTO_PARTITIONING_STRATEGY_SCALE_UP = 2;
// The auto partitioning algorithm will both increase and decrease partitions count depending on the load characteristics.
AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN = 3;
// The auto partitioning is paused.
AUTO_PARTITIONING_STRATEGY_PAUSED = 4;
}

// Partitioning settings for topic.
message PartitioningSettings {
// Minimum partition count auto merge would stop working at.
// Auto merge would stop working when the partitions count reaches min_active_partitions
// Zero value means default - 1.
int64 min_active_partitions = 1 [(Ydb.value) = ">= 0"];
// Auto split would stop working when the partitions count reaches max_active_partitions
// Zero value means default - 1.
int64 max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
int64 partition_count_limit = 2 [(Ydb.value) = ">= 0"];
// Use max_active_partitions
int64 partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
// Settings for the partitions count auto partitioning.
AutoPartitioningSettings auto_partitioning_settings = 4;
}

message AutoPartitioningSettings {
// Strategy of auto partitioning.
AutoPartitioningStrategy strategy = 1;
// Partition write speed auto partitioning options.
AutoPartitioningWriteSpeedStrategy partition_write_speed = 2;
}

message AutoPartitioningWriteSpeedStrategy {
//Partition will be auto partitioned up (divided into 2 partitions)
//after write speed to the partition exceeds up_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window

//Partition will become a candidate to the auto partitioned down
//after write speed doesn’t reach down_utilization_percent (in percentage of maximum write speed to the partition) for the period of time stabilization_window
//This candidate partition will be auto partitioned down when other neighbour partition will become a candidate to the auto partitioning down and not earlier than a retention period.

// Zero value means default - 300.
google.protobuf.Duration stabilization_window = 1;
// Zero value means default - 90.
int32 up_utilization_percent = 2 [(Ydb.value) = ">= 0"];
// Zero value means default - 30.
int32 down_utilization_percent = 3 [(Ydb.value) = ">= 0"];
}

// Partitioning settings for topic.
message AlterPartitioningSettings {
// Minimum partition count auto merge would stop working at.
// Zero value means default - 1.
optional int64 set_min_active_partitions = 1 [(Ydb.value) = ">= 0"];
// Maximum partition count auto merge would stop working at.
// Zero value means default - 1.
optional int64 set_max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0"];
// Use set_max_active_partitions
optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
// Settings for auto partitioning the partition number
optional AlterAutoPartitioningSettings alter_auto_partitioning_settings = 4;
}

message AlterAutoPartitioningSettings {
// Strategy of auto partitioning
optional AutoPartitioningStrategy set_strategy = 1;
// Auto partitioning write speed options.
optional AlterAutoPartitioningWriteSpeedStrategy set_partition_write_speed = 2;
}

message AlterAutoPartitioningWriteSpeedStrategy {
// The time of exceeding the threshold value, after which the partition will be
// auto partitioning.
// Zero value means default - 300.
optional google.protobuf.Duration set_stabilization_window = 1;
// The threshold value of the write speed to the partition as a percentage, when exceeded,
// the partition will be auto split.
// Zero value means default - 90.
optional int32 set_up_utilization_percent = 2 [(Ydb.value) = ">= 0"];
// The threshold value of the write speed to the partition as a percentage, if it is not reached,
// the partition will be auto merged.
// Zero value means default - 30.
optional int32 set_down_utilization_percent = 3 [(Ydb.value) = ">= 0"];
}

// Metering mode specifies the method used to determine consumption of resources by the topic.
Expand Down Expand Up @@ -920,6 +1026,13 @@ message DescribeTopicResponse {
Ydb.Operations.Operation operation = 1;
}

message PartitionKeyRange {
// Inclusive left border. Emptiness means -inf.
optional bytes from_bound = 1;
// Exclusive right border. Emptiness means +inf.
optional bytes to_bound = 2;
}

// Describe topic result message that will be inside DescribeTopicResponse.operation.
message DescribeTopicResult {
// Description of scheme object.
Expand Down Expand Up @@ -988,6 +1101,8 @@ message DescribeTopicResult {

// Partition location, filled only when include_location in request is true.
PartitionLocation partition_location = 6;

PartitionKeyRange key_range = 7;
}

message TopicStats {
Expand Down Expand Up @@ -1027,7 +1142,7 @@ message DescribePartitionResponse {
Ydb.Operations.Operation operation = 1;
}

// Describe partition result message that will be inside DescribeTopicResponse.operation.
// Describe partition result message that will be inside DescribePartitionResponse.operation.
message DescribePartitionResult {
// Partitions description.
DescribeTopicResult.PartitionInfo partition = 1;
Expand Down Expand Up @@ -1200,4 +1315,4 @@ message DropTopicResponse {

// Drop topic result message that will be inside DropTopicResponse.operation.
message DropTopicResult {
}
}