From 6ee96960937dd85c396a926b612dc6ac500046dd Mon Sep 17 00:00:00 2001 From: vlad-gogov Date: Thu, 19 Dec 2024 12:07:17 +0000 Subject: [PATCH] default compression via CS config --- ydb/core/formats/arrow/serializer/native.h | 19 ++- ydb/core/kqp/host/kqp_gateway_proxy.cpp | 9 +- ydb/core/kqp/ut/common/columnshard.cpp | 30 +++-- ydb/core/kqp/ut/common/columnshard.h | 2 +- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 121 +++++++++++++++--- ydb/core/protos/config.proto | 2 + .../olap/column_families/schema.cpp | 35 ++--- .../olap/column_families/update.cpp | 65 ++++++---- .../schemeshard/olap/column_families/update.h | 2 +- .../olap/operations/create_table.cpp | 5 +- 10 files changed, 208 insertions(+), 82 deletions(-) diff --git a/ydb/core/formats/arrow/serializer/native.h b/ydb/core/formats/arrow/serializer/native.h index d09241a7799f..9318a04294e2 100644 --- a/ydb/core/formats/arrow/serializer/native.h +++ b/ydb/core/formats/arrow/serializer/native.h @@ -1,11 +1,16 @@ #pragma once #include "abstract.h" +#include "parsing.h" +#include "utils.h" +#include +#include #include #include #include +#include #include #include @@ -53,7 +58,19 @@ class TNativeSerializer: public ISerializer { static arrow::ipc::IpcOptions BuildDefaultOptions() { arrow::ipc::IpcWriteOptions options; options.use_threads = false; - options.codec = *arrow::util::Codec::Create(arrow::Compression::LZ4_FRAME); + std::optional codec = CompressionFromProto(AppData()->ColumnShardConfig.GetDefaultCompression()); + Y_VERIFY_S(codec.has_value(), "ColumnShardConfig: Unknown compression"); + if (AppData()->ColumnShardConfig.HasDefaultCompressionLevel()) { + TString errorMessage = TStringBuilder() << "ColumnShardConfig: Compression `" << CompressionToString(codec.value()) + << "` is not support compression level"; + Y_VERIFY_S(SupportsCompressionLevel(codec.value()), errorMessage.c_str()); + } + if (codec.value() == arrow::Compression::type::ZSTD) { + i32 codecLevel = AppData()->ColumnShardConfig.GetDefaultCompressionLevel(); + options.codec = NArrow::TStatusValidator::GetValid(arrow::util::Codec::Create(codec.value(), codecLevel)); + } else { + options.codec = NArrow::TStatusValidator::GetValid(arrow::util::Codec::Create(codec.value())); + } return options; } diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 81ceb584d6ba..1330592e42db 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -415,13 +415,14 @@ bool FillColumnTableSchema(NKikimrSchemeOp::TColumnTableSchema& schema, const T& return false; } familyDescription->SetColumnCodec(codec); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Compression is not set for column family'" << family.Name << "'"; - return false; } if (family.CompressionLevel.Defined()) { + if (!family.Compression.Defined()) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Compression is not set for column family '" << family.Name << "', but compression level is set"; + return false; + } familyDescription->SetColumnCodecLevel(family.CompressionLevel.GetRef()); } } diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 28ffe5f3e7dd..485a6a02ac62 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -160,7 +160,9 @@ namespace NKqp { TString TTestHelper::TCompression::BuildQuery() const { TStringBuilder str; - str << "COMPRESSION=\"" << NArrow::CompressionToString(CompressionType) << "\""; + if (CompressionType.has_value()) { + str << "COMPRESSION=\"" << NArrow::CompressionToString(CompressionType.value()) << "\""; + } if (CompressionLevel.has_value()) { str << ", COMPRESSION_LEVEL=" << CompressionLevel.value(); } @@ -173,9 +175,16 @@ namespace NKqp { << "` and in right value `" << rhs.GetSerializerClassName() << "`"; return false; } - if (CompressionType != rhs.GetCompressionType()) { - errorMessage = TStringBuilder() << "different compression type: in left value `" << NArrow::CompressionToString(CompressionType) - << "` and in right value `" << NArrow::CompressionToString(rhs.GetCompressionType()) << "`"; + if (CompressionType.has_value() && rhs.GetCompressionType().has_value() && CompressionType.value() != rhs.GetCompressionType().value()) { + errorMessage = TStringBuilder() << "different compression type: in left value `" + << NArrow::CompressionToString(CompressionType.value()) << "` and in right value `" + << NArrow::CompressionToString(rhs.GetCompressionType().value()) << "`"; + return false; + } else if (CompressionType.has_value() && !rhs.GetCompressionType().has_value()) { + errorMessage = TStringBuilder() << "compression type is set in left value, but not set in right value"; + return false; + } else if (!CompressionType.has_value() && rhs.GetCompressionType().has_value()) { + errorMessage = TStringBuilder() << "compression type is not set in left value, but set in right value"; return false; } if (CompressionLevel.has_value() && rhs.GetCompressionLevel().has_value() && @@ -199,12 +208,15 @@ namespace NKqp { } bool TTestHelper::TColumnFamily::DeserializeFromProto(const NKikimrSchemeOp::TFamilyDescription& family) { - if (!family.HasId() || !family.HasName() || !family.HasColumnCodec()) { + if (!family.HasId() || !family.HasName()) { return false; } Id = family.GetId(); FamilyName = family.GetName(); - Compression = TTestHelper::TCompression().SetCompressionType(family.GetColumnCodec()); + Compression = TTestHelper::TCompression(); + if (family.HasColumnCodec()) { + Compression.SetCompressionType(family.GetColumnCodec()); + } if (family.HasColumnCodecLevel()) { Compression.SetCompressionLevel(family.GetColumnCodecLevel()); } @@ -296,9 +308,9 @@ namespace NKqp { TString TTestHelper::TColumnTableBase::BuildAlterCompressionQuery(const TString& columnName, const TCompression& compression) const { auto str = TStringBuilder() << "ALTER OBJECT `" << Name << "` (TYPE " << GetObjectType() << ") SET"; str << " (ACTION=ALTER_COLUMN, NAME=" << columnName << ", `SERIALIZER.CLASS_NAME`=`" << compression.GetSerializerClassName() << "`,"; - auto codec = NArrow::CompressionFromProto(compression.GetCompressionType()); - Y_VERIFY(codec.has_value()); - str << " `COMPRESSION.TYPE`=`" << NArrow::CompressionToString(codec.value()) << "`"; + if (compression.GetCompressionType().has_value()) { + str << " `COMPRESSION.TYPE`=`" << NArrow::CompressionToString(compression.GetCompressionType().value()) << "`"; + } if (compression.GetCompressionLevel().has_value()) { str << "`COMPRESSION.LEVEL`=" << compression.GetCompressionLevel().value(); } diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h index ba85f6686162..e3e9d676729e 100644 --- a/ydb/core/kqp/ut/common/columnshard.h +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -20,7 +20,7 @@ class TTestHelper { public: class TCompression { YDB_ACCESSOR(TString, SerializerClassName, "ARROW_SERIALIZER"); - YDB_ACCESSOR_DEF(NKikimrSchemeOp::EColumnCodec, CompressionType); + YDB_ACCESSOR_DEF(std::optional, CompressionType); YDB_ACCESSOR_DEF(std::optional, CompressionLevel); public: diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index f200071b8ebe..ab4722a87933 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -9468,10 +9468,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { auto describeResult = DescribeTable(&runner.GetTestServer(), sender, tableName); auto schema = describeResult.GetPathDescription().GetColumnTableDescription().GetSchema(); - TTestHelper::TCompression plainCompression = - TTestHelper::TCompression().SetCompressionType(NKikimrSchemeOp::EColumnCodec::ColumnCodecPlain); - TTestHelper::TColumnFamily defaultFamily = - TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default").SetCompression(plainCompression); + TTestHelper::TColumnFamily defaultFamily = TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default"); UNIT_ASSERT_EQUAL(schema.ColumnFamiliesSize(), 1); TTestHelper::TColumnFamily defaultFromScheme; @@ -9480,18 +9477,6 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TString errorMessage; UNIT_ASSERT_C(defaultFromScheme.IsEqual(defaultFamily, errorMessage), errorMessage); } - - auto columns = schema.GetColumns(); - for (ui32 i = 0; i < schema.ColumnsSize(); i++) { - auto column = columns[i]; - UNIT_ASSERT(column.HasSerializer()); - UNIT_ASSERT_EQUAL_C( - column.GetColumnFamilyId(), 0, TStringBuilder() << "family for column " << column.GetName() << " is not default"); - TTestHelper::TCompression compression; - UNIT_ASSERT(compression.DeserializeFromProto(schema.GetColumns(i).GetSerializer())); - TString errorMessage; - UNIT_ASSERT_C(compression.IsEqual(defaultFamily.GetCompression(), errorMessage), errorMessage); - } } // Field `Data` is not used in ColumnFamily for ColumnTable @@ -10263,11 +10248,10 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper testHelper(TKikimrSettings().SetWithSampleTables(false)); TString tableName = "/Root/TableWithFamily"; - TTestHelper::TCompression plainCompression = - TTestHelper::TCompression().SetCompressionType(NKikimrSchemeOp::EColumnCodec::ColumnCodecPlain); + TTestHelper::TCompression lz4Compression = TTestHelper::TCompression().SetCompressionType(NKikimrSchemeOp::EColumnCodec::ColumnCodecLZ4); TVector families = { - TTestHelper::TColumnFamily().SetId(1).SetFamilyName("family1").SetCompression(plainCompression), + TTestHelper::TColumnFamily().SetId(1).SetFamilyName("family1").SetCompression(lz4Compression), }; { @@ -10290,7 +10274,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.CreateTable(testTable); } - families.push_back(TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default").SetCompression(plainCompression)); + families.push_back(TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default")); auto& runner = testHelper.GetKikimr(); auto runtime = runner.GetTestServer().GetRuntime(); TActorId sender = runtime->AllocateEdgeActor(); @@ -10715,6 +10699,103 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.RebootTablets("/Root/ColumnTableTest"); } + Y_UNIT_TEST(CreateTableWithDefaultFamilyWithoutSettings) { + TTestHelper testHelper(TKikimrSettings().SetWithSampleTables(false)); + TString tableName = "/Root/ColumnTableTest"; + auto session = testHelper.GetSession(); + auto createQuery = TStringBuilder() << R"(CREATE TABLE `)" << tableName << R"(` ( + Key Uint64 NOT NULL, + Value1 String, + Value2 Uint32, + PRIMARY KEY (Key), + FAMILY default ()) + WITH (STORE = COLUMN);)"; + auto result = session.ExecuteSchemeQuery(createQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto& runner = testHelper.GetKikimr(); + auto runtime = runner.GetTestServer().GetRuntime(); + TActorId sender = runtime->AllocateEdgeActor(); + + auto describeResult = DescribeTable(&runner.GetTestServer(), sender, tableName); + auto schema = describeResult.GetPathDescription().GetColumnTableDescription().GetSchema(); + TTestHelper::TColumnFamily defaultFamily = TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default"); + + UNIT_ASSERT_EQUAL(schema.ColumnFamiliesSize(), 1); + TTestHelper::TColumnFamily defaultFromScheme; + UNIT_ASSERT(defaultFromScheme.DeserializeFromProto(schema.GetColumnFamilies(0))); + { + TString errorMessage; + UNIT_ASSERT_C(defaultFromScheme.IsEqual(defaultFamily, errorMessage), errorMessage); + } + } + + Y_UNIT_TEST(CreateTableWithFamilyWithOnlyCompressionLevel) { + TTestHelper testHelper(TKikimrSettings().SetWithSampleTables(false)); + TString tableName = "/Root/ColumnTableTest"; + auto session = testHelper.GetSession(); + auto createQuery = TStringBuilder() << R"(CREATE TABLE `)" << tableName << R"(` ( + Key Uint64 NOT NULL, + Value1 String, + Value2 Uint32, + PRIMARY KEY (Key), + FAMILY family1 ( + COMPRESSION_LEVEL = 2 + )) + WITH (STORE = COLUMN);)"; + auto result = session.ExecuteSchemeQuery(createQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + + void CreateTableWithCompressionSettings(const TTestHelper::TCompression& compression, const EStatus& statusCreateTable = EStatus::SUCCESS) { + NKikimrConfig::TAppConfig appConfig; + auto* CSConfig = appConfig.MutableColumnShardConfig(); + CSConfig->SetDefaultCompression(compression.GetCompressionType().value()); + if (compression.GetCompressionLevel().has_value()) { + CSConfig->SetDefaultCompressionLevel(compression.GetCompressionLevel().value()); + } + TTestHelper testHelper(TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appConfig)); + + TString tableName = "/Root/TableWithoutColumnFamily"; + { + TVector schema = { + TTestHelper::TColumnSchema().SetName("Key").SetType(NScheme::NTypeIds::Uint64).SetNullable(false), + TTestHelper::TColumnSchema().SetName("Value1").SetType(NScheme::NTypeIds::String).SetNullable(true), + TTestHelper::TColumnSchema().SetName("Value2").SetType(NScheme::NTypeIds::Uint32).SetNullable(true) + }; + + TTestHelper::TColumnTable testTable; + testTable.SetName(tableName).SetPrimaryKey({ "Key" }).SetSchema(schema); + testHelper.CreateTable(testTable, statusCreateTable); + } + + if (statusCreateTable == EStatus::SUCCESS) { + auto& runner = testHelper.GetKikimr(); + auto tableClient = runner.GetTableClient(); + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + + auto runtime = runner.GetTestServer().GetRuntime(); + TActorId sender = runtime->AllocateEdgeActor(); + + auto describeResult = DescribeTable(&runner.GetTestServer(), sender, tableName); + auto schema = describeResult.GetPathDescription().GetColumnTableDescription().GetSchema(); + TTestHelper::TColumnFamily defaultFamily = TTestHelper::TColumnFamily().SetId(0).SetFamilyName("default"); + + UNIT_ASSERT_EQUAL(schema.ColumnFamiliesSize(), 1); + TTestHelper::TColumnFamily defaultFromScheme; + UNIT_ASSERT(defaultFromScheme.DeserializeFromProto(schema.GetColumnFamilies(0))); + { + TString errorMessage; + UNIT_ASSERT_C(defaultFromScheme.IsEqual(defaultFamily, errorMessage), errorMessage); + } + } + } + + Y_UNIT_TEST(DefaultCompressionViaConfig) { + CreateTableWithCompressionSettings(TTestHelper::TCompression().SetCompressionType(NKikimrSchemeOp::EColumnCodec::ColumnCodecLZ4)); + CreateTableWithCompressionSettings( + TTestHelper::TCompression().SetCompressionType(NKikimrSchemeOp::EColumnCodec::ColumnCodecZSTD).SetCompressionLevel(5)); + } } Y_UNIT_TEST_SUITE(KqpOlapTypes) { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index b2a2ca09d088..4c9c5fb2317e 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1736,6 +1736,8 @@ message TColumnShardConfig { optional string ReaderClassName = 28; optional bool AllowNullableColumnsInPK = 29 [default = false]; optional uint32 RestoreDataOnWriteTimeoutSeconds = 30; + optional NKikimrSchemeOp.EColumnCodec DefaultCompression = 31 [default = ColumnCodecZSTD]; + optional int32 DefaultCompressionLevel = 32 [default = 1]; } message TSchemeShardConfig { diff --git a/ydb/core/tx/schemeshard/olap/column_families/schema.cpp b/ydb/core/tx/schemeshard/olap/column_families/schema.cpp index 7667e984d532..875f5678e1e0 100644 --- a/ydb/core/tx/schemeshard/olap/column_families/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/column_families/schema.cpp @@ -119,23 +119,24 @@ bool TOlapColumnFamiliesDescription::Validate(const NKikimrSchemeOp::TColumnTabl } lastColumnFamilyId = familyProto.GetId(); - if (!familyProto.HasColumnCodec()) { - errors.AddError("missing column codec for column family '" + columnFamilyName + "'"); - return false; - } - - auto serializerProto = ConvertFamilyDescriptionToProtoSerializer(familyProto); - if (serializerProto.IsFail()) { - errors.AddError(serializerProto.GetErrorMessage()); - return false; - } - NArrow::NSerialization::TSerializerContainer serializer; - if (!serializer.DeserializeFromProto(serializerProto.GetResult())) { - errors.AddError(TStringBuilder() << "can't deserialize column family `" << columnFamilyName << "` from proto "); - return false; - } - if (!family->GetSerializerContainer().IsEqualTo(serializer)) { - errors.AddError(TStringBuilder() << "compression from column family '" << columnFamilyName << "` is not matching schema preset"); + if (familyProto.HasColumnCodec() && family->GetSerializerContainer().has_value()) { + auto serializerProto = ConvertFamilyDescriptionToProtoSerializer(familyProto); + if (serializerProto.IsFail()) { + errors.AddError(serializerProto.GetErrorMessage()); + return false; + } + NArrow::NSerialization::TSerializerContainer serializer; + if (!serializer.DeserializeFromProto(serializerProto.GetResult())) { + errors.AddError(TStringBuilder() << "can't deserialize column family `" << columnFamilyName << "` from proto "); + return false; + } + if (!family->GetSerializerContainer()->IsEqualTo(serializer)) { + errors.AddError(TStringBuilder() << "compression from column family '" << columnFamilyName << "` is not matching schema preset"); + return false; + } + } else if ((!familyProto.HasColumnCodec() && family->GetSerializerContainer().has_value()) || + (familyProto.HasColumnCodec() && !family->GetSerializerContainer().has_value())) { + errors.AddError(TStringBuilder() << "compression is not matching schema preset in column family `" << columnFamilyName << "`"); return false; } } diff --git a/ydb/core/tx/schemeshard/olap/column_families/update.cpp b/ydb/core/tx/schemeshard/olap/column_families/update.cpp index 9460f8f1be0d..6f7918e19a59 100644 --- a/ydb/core/tx/schemeshard/olap/column_families/update.cpp +++ b/ydb/core/tx/schemeshard/olap/column_families/update.cpp @@ -87,55 +87,66 @@ bool TOlapColumnFamlilyAdd::ParseFromRequest(const NKikimrSchemeOp::TFamilyDescr } Name = columnFamily.GetName(); - auto serializer = ConvertFamilyDescriptionToProtoSerializer(columnFamily); - if (serializer.IsFail()) { - errors.AddError(serializer.GetErrorMessage()); - return false; - } - auto resultBuild = NArrow::NSerialization::TSerializerContainer::BuildFromProto(serializer.GetResult()); - if (resultBuild.IsFail()) { - errors.AddError(resultBuild.GetErrorMessage()); - return false; + if (columnFamily.HasColumnCodec()) { + auto serializer = ConvertFamilyDescriptionToProtoSerializer(columnFamily); + if (serializer.IsFail()) { + errors.AddError(serializer.GetErrorMessage()); + return false; + } + auto resultBuild = NArrow::NSerialization::TSerializerContainer::BuildFromProto(serializer.GetResult()); + if (resultBuild.IsFail()) { + errors.AddError(resultBuild.GetErrorMessage()); + return false; + } + SerializerContainer = resultBuild.GetResult(); } - SerializerContainer = resultBuild.GetResult(); return true; } void TOlapColumnFamlilyAdd::ParseFromLocalDB(const NKikimrSchemeOp::TFamilyDescription& columnFamily) { Name = columnFamily.GetName(); - auto serializer = ConvertFamilyDescriptionToProtoSerializer(columnFamily); - Y_VERIFY_S(serializer.IsSuccess(), serializer.GetErrorMessage()); - Y_VERIFY(SerializerContainer.DeserializeFromProto(serializer.GetResult())); + if (columnFamily.HasColumnCodec()) { + auto serializer = ConvertFamilyDescriptionToProtoSerializer(columnFamily); + Y_VERIFY_S(serializer.IsSuccess(), serializer.GetErrorMessage()); + SerializerContainer = NArrow::NSerialization::TSerializerContainer(); + Y_VERIFY(SerializerContainer->DeserializeFromProto(serializer.GetResult())); + } } void TOlapColumnFamlilyAdd::Serialize(NKikimrSchemeOp::TFamilyDescription& columnFamily) const { - auto result = ConvertSerializerContainerToFamilyDescription(SerializerContainer); - Y_VERIFY_S(result.IsSuccess(), result.GetErrorMessage()); columnFamily.SetName(Name); - columnFamily.SetColumnCodec(result->GetColumnCodec()); - if (result->HasColumnCodecLevel()) { - columnFamily.SetColumnCodecLevel(result->GetColumnCodecLevel()); + if (SerializerContainer.has_value()) { + auto result = ConvertSerializerContainerToFamilyDescription(SerializerContainer.value()); + Y_VERIFY_S(result.IsSuccess(), result.GetErrorMessage()); + columnFamily.SetColumnCodec(result->GetColumnCodec()); + if (result->HasColumnCodecLevel()) { + columnFamily.SetColumnCodecLevel(result->GetColumnCodecLevel()); + } } } bool TOlapColumnFamlilyAdd::ApplyDiff(const TOlapColumnFamlilyDiff& diffColumnFamily, IErrorCollector& errors) { Y_ABORT_UNLESS(GetName() == diffColumnFamily.GetName()); - auto newColumnFamily = ConvertSerializerContainerToFamilyDescription(SerializerContainer); - if (newColumnFamily.IsFail()) { - errors.AddError(newColumnFamily.GetErrorMessage()); - return false; + NKikimrSchemeOp::TFamilyDescription newColumnFamily; + if (SerializerContainer.has_value()) { + auto resultConvert = ConvertSerializerContainerToFamilyDescription(SerializerContainer.value()); + if (resultConvert.IsFail()) { + errors.AddError(resultConvert.GetErrorMessage()); + return false; + } + newColumnFamily = resultConvert.GetResult(); } - newColumnFamily->SetName(GetName()); + newColumnFamily.SetName(GetName()); auto codec = diffColumnFamily.GetCodec(); if (codec.has_value()) { - newColumnFamily->SetColumnCodec(codec.value()); - newColumnFamily->ClearColumnCodecLevel(); + newColumnFamily.SetColumnCodec(codec.value()); + newColumnFamily.ClearColumnCodecLevel(); } auto codecLevel = diffColumnFamily.GetCodecLevel(); if (codecLevel.has_value()) { - newColumnFamily->SetColumnCodecLevel(codecLevel.value()); + newColumnFamily.SetColumnCodecLevel(codecLevel.value()); } - auto serializer = ConvertFamilyDescriptionToProtoSerializer(newColumnFamily.GetResult()); + auto serializer = ConvertFamilyDescriptionToProtoSerializer(newColumnFamily); if (serializer.IsFail()) { errors.AddError(serializer.GetErrorMessage()); return false; diff --git a/ydb/core/tx/schemeshard/olap/column_families/update.h b/ydb/core/tx/schemeshard/olap/column_families/update.h index 63c0e53af16d..171567f411f7 100644 --- a/ydb/core/tx/schemeshard/olap/column_families/update.h +++ b/ydb/core/tx/schemeshard/olap/column_families/update.h @@ -23,7 +23,7 @@ class TOlapColumnFamlilyDiff { class TOlapColumnFamlilyAdd { private: YDB_READONLY_DEF(TString, Name); - YDB_READONLY_DEF(NArrow::NSerialization::TSerializerContainer, SerializerContainer); + YDB_READONLY_DEF(std::optional, SerializerContainer); public: bool ParseFromRequest(const NKikimrSchemeOp::TFamilyDescription& columnFamily, IErrorCollector& errors); diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp index c69ab804eb35..0c81af0a651a 100644 --- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp @@ -561,7 +561,9 @@ class TCreateColumnTable: public TSubOperation { void AddDefaultFamilyIfNotExists(NKikimrSchemeOp::TColumnTableDescription& createDescription) { auto schema = createDescription.GetSchema(); - for (const auto& family : schema.GetColumnFamilies()) { + + for (ui32 i = 0; i < schema.ColumnFamiliesSize(); i++) { + auto family = schema.GetColumnFamilies(i); if (family.GetName() == "default") { return; } @@ -571,7 +573,6 @@ class TCreateColumnTable: public TSubOperation { auto defaultFamily = mutableSchema->AddColumnFamilies(); defaultFamily->SetName("default"); defaultFamily->SetId(0); - defaultFamily->SetColumnCodec(NKikimrSchemeOp::EColumnCodec::ColumnCodecPlain); for (ui32 i = 0; i < schema.ColumnsSize(); i++) { if (!schema.GetColumns(i).HasColumnFamilyName() || !schema.GetColumns(i).HasColumnFamilyId()) {