diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/App.config b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/App.config new file mode 100644 index 0000000..c414bbb --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/App.config @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/ClickHouseClient.CLI.csproj b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/ClickHouseClient.CLI.csproj new file mode 100644 index 0000000..187712c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/ClickHouseClient.CLI.csproj @@ -0,0 +1,61 @@ + + + + + Debug + AnyCPU + {D3D02050-BD31-460D-8087-76D98A226F78} + Exe + YPermitin.SQLCLR.ClickHouseClient.CLI + ClickHouseClient.CLI + v4.8 + 512 + true + true + + 9.0 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + {ac5f49bf-4526-47a3-8034-75f84108cee5} + ClickHouseClient.Entry + + + + \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Program.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Program.cs new file mode 100644 index 0000000..ee7dfac --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Program.cs @@ -0,0 +1,231 @@ +using System; +using System.Data; +using System.Data.SqlClient; +using YPermitin.SQLCLR.ClickHouseClient.Entry; +using YPermitin.SQLCLR.ClickHouseClient.Entry.Extensions; +using YPermitin.SQLCLR.ClickHouseClient.Models; + +namespace ClickHouseClient.CLI +{ + internal class Program + { + static void Main(string[] args) + { + Console.WriteLine("Начало проверки работы с ClickHouse."); + + // Строка подключения к SQL Server + EntryBase.ConnectionString = @"server=localhost;database=master;trusted_connection=true;"; + // Строка подключения к ClickHouse + string clickHouseConnectionString = @"Host=yy-comp;Port=8123;Username=default;password=;Database=default;"; + + Console.WriteLine("Строка подключения SQL Server: {0}", EntryBase.ConnectionString); + Console.WriteLine("Строка подключения ClickHouse: {0}", clickHouseConnectionString); + + Console.Write("Установка соединения с SQL Server..."); + // Создаем соединение с SQL Server для дальнейшей работы + using (SqlConnection sqlConnection = new SqlConnection(EntryBase.ConnectionString)) + { + sqlConnection.Open(); + Console.WriteLine("OK!"); + // Устанавливаем подключение для отладки + EntryBase.DebugConnection = sqlConnection; + + #region ExecuteScalar + + Console.WriteLine("Начало работы метода ExecuteScalar."); + + // Выполняем запрос с возвратом одного значения + string clickHouseVersion = EntryClickHouseClient.ExecuteScalar( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: "SELECT version()".ToSqlChars()) + .ToStringFromSqlChars(); + Console.WriteLine("Версия ClickHouse: {0}", clickHouseVersion); + + Console.WriteLine("Окончание работы метода ExecuteScalar."); + + #endregion + + #region ExecuteStatement + + Console.WriteLine("Начало работы метода ExecuteStatement."); + + // Выполняем запрос без возврата результата. + // В качестве примера создаем таблицу, предварительно удалив, если она существовала. + + Console.WriteLine("Удаляем существующую таблицу, если она существует."); + EntryClickHouseClient.ExecuteStatement( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: @" +DROP TABLE IF EXISTS SimpleTable +".ToSqlChars()); + + Console.WriteLine("Создаем таблицу."); + EntryClickHouseClient.ExecuteStatement( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: @" +CREATE TABLE IF NOT EXISTS SimpleTable +( + Id UInt64, + Period datetime DEFAULT now(), + Name String +) +ENGINE = MergeTree +ORDER BY Id; +".ToSqlChars()); + + // А затем вставляем 100 записей + Console.WriteLine("Добавленые новых записей...."); + for (int i = 1; i <= 10; i++) + { + string rowName = "Row " + i; + EntryClickHouseClient.ExecuteStatement( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: (@" +INSERT INTO SimpleTable +( + Id, + Name +) +VALUES(" + i + @", '" + rowName + @"'); +").ToSqlChars() + ); + + Console.WriteLine("Добавлена запись {0} - {1}", i, rowName); + } + + Console.WriteLine("Окончание работы метода ExecuteStatement."); + + #endregion + + #region ExecuteSimple + + Console.WriteLine("Начало работы метода ExecuteSimple."); + + // Выполняем просто запрос с возвратом результата. + // У этого метода одно ограничение - возвращается только первая колонока запроса SELECT + // и только в виде строки. + // Для возвращения нескольких колонок можно возвращать кортеж, который будет преобразован к JSON-строке. + + var simpleQueryResult = EntryClickHouseClient.ExecuteSimple( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: @" +SELECT + tuple(name, engine, data_path) +FROM `system`.`databases` +".ToSqlChars()); + + var enumerator = simpleQueryResult.GetEnumerator(); + int rowsCount = 0; + while (enumerator.MoveNext()) + { + Console.WriteLine(((ExecuteSimpleRowResult)enumerator.Current).ResultValue); + rowsCount++; + } + + Console.WriteLine("Количество записей из результата запроса: {0}.", rowsCount); + + Console.WriteLine("Окончание работы метода ExecuteSimple."); + + #endregion + + #region ExecuteToTempTable + + Console.WriteLine("Начало работы метода ExecuteToTempTable."); + + // Создаем временную таблицу для сохранения результата запроса из ClickHouse + Console.WriteLine("Создаем временную таблицу."); + string sqlCreateTempTable = @" +IF(OBJECT_ID('tempdb..#logs') IS NOT NULL) + DROP TABLE #logs; +CREATE TABLE #logs +( + [EventTime] datetime2(0), + [Query] nvarchar(max), + [Tables] nvarchar(max), + [QueryId] uniqueidentifier +); +"; + SqlCommand sqlCreateTempTableCommand = new SqlCommand(sqlCreateTempTable, sqlConnection); + sqlCreateTempTableCommand.CommandType = System.Data.CommandType.Text; + sqlCreateTempTableCommand.ExecuteNonQuery(); + + // Выполняем запрос к ClickHouse и сохраняем во временную таблицу + Console.WriteLine("Выполняем запрос к ClickHouse и сохраняем результат во временную таблицу."); + EntryClickHouseClient.ExecuteToTempTable( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: @" +select + event_time, + query, + tables, + query_id +from `system`.query_log +limit 1000 +".ToSqlChars(), + tempTableName: "#logs".ToSqlChars()); + + int totalRows = 0; + SqlCommand sqlTempTableRows = new SqlCommand("SELECT COUNT(*) FROM #logs", sqlConnection); + sqlTempTableRows.CommandType = System.Data.CommandType.Text; + totalRows = (int)sqlTempTableRows.ExecuteScalar(); + Console.WriteLine("Всего записей прочитано: {0}", totalRows); + + Console.WriteLine("Окончание работы метода ExecuteToTempTable."); + + #endregion + + #region ExecuteToGlobalTempTable + + Console.WriteLine("Начало работы метода ExecuteToGlobalTempTable."); + + // Создаем временную таблицу для сохранения результата запроса из ClickHouse + Console.WriteLine("Создаем ГЛОБАЛЬНУЮ временную таблицу."); + string sqlCreateGlobalTempTable = @" +IF(OBJECT_ID('tempdb..##logs') IS NOT NULL) + DROP TABLE #logs; +CREATE TABLE ##logs +( + [EventTime] datetime2(0), + [Query] nvarchar(max), + [Tables] nvarchar(max), + [QueryId] uniqueidentifier +); +"; + SqlCommand sqlCreateGlobalTempTableCommand = new SqlCommand(sqlCreateGlobalTempTable, sqlConnection); + sqlCreateTempTableCommand.CommandType = System.Data.CommandType.Text; + sqlCreateTempTableCommand.ExecuteNonQuery(); + + // Выполняем запрос к ClickHouse и сохраняем во временную таблицу + Console.WriteLine("Выполняем запрос к ClickHouse и сохраняем результат во временную таблицу."); + EntryClickHouseClient.ExecuteToGlobalTempTable( + connectionString: clickHouseConnectionString.ToSqlChars(), + queryText: @" +select + event_time, + query, + tables, + query_id +from `system`.query_log +limit 10 +".ToSqlChars(), + tempTableName: "##logs".ToSqlChars(), + sqlServerConnectionString: EntryBase.ConnectionString.ToSqlChars()); + + int totalRowsGlobal = 0; + SqlCommand sqlTempTableRowsGlobal = new SqlCommand("SELECT COUNT(*) FROM ##logs", sqlConnection); + sqlTempTableRowsGlobal.CommandType = System.Data.CommandType.Text; + totalRowsGlobal = (int)sqlTempTableRowsGlobal.ExecuteScalar(); + Console.WriteLine("Всего записей прочитано: {0}", totalRowsGlobal); + + Console.WriteLine("Окончание работы метода ExecuteToGlobalTempTable."); + + #endregion + } + + // Очищаем отладочное соединение SQL Server + EntryBase.DebugConnection = null; + + Console.WriteLine("Окончание проверки работы с ClickHouse."); + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Properties/AssemblyInfo.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..7fd5b57 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Apps/ClickHouseClient.CLI/Properties/AssemblyInfo.cs @@ -0,0 +1,33 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// Общие сведения об этой сборке предоставляются следующим набором +// набора атрибутов. Измените значения этих атрибутов для изменения сведений, +// связанные с этой сборкой. +[assembly: AssemblyTitle("ClickHouseClient.CLI")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ClickHouseClient.CLI")] +[assembly: AssemblyCopyright("Copyright © 2024")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Установка значения False для параметра ComVisible делает типы в этой сборке невидимыми +// для компонентов COM. Если необходимо обратиться к типу в этой сборке через +// из модели COM задайте для атрибута ComVisible этого типа значение true. +[assembly: ComVisible(false)] + +// Следующий GUID представляет идентификатор typelib, если этот проект доступен из модели COM +[assembly: Guid("d3d02050-bd31-460d-8087-76d98a226f78")] + +// Сведения о версии сборки состоят из указанных ниже четырех значений: +// +// Основной номер версии +// Дополнительный номер версии +// Номер сборки +// Номер редакции +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/ClickHouseClient.sln b/SQL-Server-SQLCLR/Projects/ClickHouseClient/ClickHouseClient.sln new file mode 100644 index 0000000..ff1b929 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/ClickHouseClient.sln @@ -0,0 +1,50 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Apps", "Apps", "{7C642548-3BED-4C89-8070-1DAFFE063F48}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Libs", "Libs", "{8A38C4D0-173F-4C72-84E8-BCC94AED1551}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{0324CCD0-9711-4209-84E7-3B28062B2328}" + ProjectSection(SolutionItems) = preProject + Readme.md = Readme.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{0E777D3D-82EF-4E67-9EF0-7804AEBDC3C6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClickHouseClient.CLI", "Apps\ClickHouseClient.CLI\ClickHouseClient.CLI.csproj", "{D3D02050-BD31-460D-8087-76D98A226F78}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClickHouseClient", "Libs\ClickHouseClient\ClickHouseClient.csproj", "{DE8E7D56-8A86-4142-84A7-5CEB3162329F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClickHouseClient.Entry", "Libs\ClickHouseClient.Entry\ClickHouseClient.Entry.csproj", "{AC5F49BF-4526-47A3-8034-75F84108CEE5}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D3D02050-BD31-460D-8087-76D98A226F78}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3D02050-BD31-460D-8087-76D98A226F78}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3D02050-BD31-460D-8087-76D98A226F78}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3D02050-BD31-460D-8087-76D98A226F78}.Release|Any CPU.Build.0 = Release|Any CPU + {DE8E7D56-8A86-4142-84A7-5CEB3162329F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DE8E7D56-8A86-4142-84A7-5CEB3162329F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DE8E7D56-8A86-4142-84A7-5CEB3162329F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DE8E7D56-8A86-4142-84A7-5CEB3162329F}.Release|Any CPU.Build.0 = Release|Any CPU + {AC5F49BF-4526-47A3-8034-75F84108CEE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AC5F49BF-4526-47A3-8034-75F84108CEE5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AC5F49BF-4526-47A3-8034-75F84108CEE5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AC5F49BF-4526-47A3-8034-75F84108CEE5}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {D3D02050-BD31-460D-8087-76D98A226F78} = {7C642548-3BED-4C89-8070-1DAFFE063F48} + {DE8E7D56-8A86-4142-84A7-5CEB3162329F} = {8A38C4D0-173F-4C72-84E8-BCC94AED1551} + {AC5F49BF-4526-47A3-8034-75F84108CEE5} = {8A38C4D0-173F-4C72-84E8-BCC94AED1551} + EndGlobalSection +EndGlobal diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/ClickHouseClient.Entry.csproj b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/ClickHouseClient.Entry.csproj new file mode 100644 index 0000000..7105e4d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/ClickHouseClient.Entry.csproj @@ -0,0 +1,65 @@ + + + + + Debug + AnyCPU + {AC5F49BF-4526-47A3-8034-75F84108CEE5} + Library + Properties + YPermitin.SQLCLR.ClickHouseClient.Entry + ClickHouseClient.Entry + v4.8 + 512 + true + 9.0 + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\packages\Newtonsoft.Json.8.0.3\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + + + + + + + {de8e7d56-8a86-4142-84a7-5ceb3162329f} + ClickHouseClient + + + + + + + \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryBase.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryBase.cs new file mode 100644 index 0000000..2eee5da --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryBase.cs @@ -0,0 +1,23 @@ +using System.Data.SqlClient; + +namespace YPermitin.SQLCLR.ClickHouseClient.Entry +{ + public abstract class EntryBase + { + /// + /// Строка подключения к SQL Server. + /// + /// По умолчанию используется контекстное соединение, + /// из под которого выполнен вызов функции или процедуры со стороны SQL Server. + /// + public static string ConnectionString { get; set; } + = "context connection=true"; + + /// + /// Соединение SQL Server для целей отладки. + /// + /// При использовании расширения непосредственно на SQL Server не используется. + /// + public static SqlConnection DebugConnection { get; set; } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryClickHouseClient.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryClickHouseClient.cs new file mode 100644 index 0000000..56f366d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/EntryClickHouseClient.cs @@ -0,0 +1,662 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Data.SqlClient; +using System.Data.SqlTypes; +using System.Net; +using System.Security.Principal; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.SqlServer.Server; +using Newtonsoft.Json; +using YPermitin.SQLCLR.ClickHouseClient.ADO; +using YPermitin.SQLCLR.ClickHouseClient.Copy; +using YPermitin.SQLCLR.ClickHouseClient.Models; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.Entry +{ + public class EntryClickHouseClient : EntryBase + { + private static SqlChars _emptyString = new SqlChars(string.Empty); + private static readonly Dictionary> TypeConverters = new Dictionary>(); + static EntryClickHouseClient() + { + // Дата и время + TypeConverters.Add(typeof(DateTime), (sourceName, sourceType) => + { + return "datetime2(0)"; + }); + // Строковой тип + TypeConverters.Add(typeof(string), (sourceName, sourceType) => + { + return "nvarchar(max)"; + }); + // Дробное число + TypeConverters.Add(typeof(double), (sourceName, sourceType) => + { + return "numeric(35,5)"; + }); + // Дробное число + TypeConverters.Add(typeof(float), (sourceName, sourceType) => + { + return "numeric(35,5)"; + }); + // Целое число + TypeConverters.Add(typeof(Int16), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + TypeConverters.Add(typeof(Int32), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + TypeConverters.Add(typeof(UInt16), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + TypeConverters.Add(typeof(UInt32), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + // Целое число (большое) + TypeConverters.Add(typeof(Int64), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + TypeConverters.Add(typeof(UInt64), (sourceName, sourceType) => + { + return "numeric(25,0)"; + }); + // Байт + TypeConverters.Add(typeof(byte), (sourceName, sourceType) => + { + return "numeric(15,0)"; + }); + // Булево + TypeConverters.Add(typeof(bool), (sourceName, sourceType) => + { + return "bit"; + }); + // IP-адрес + TypeConverters.Add(typeof(IPAddress), (sourceName, sourceType) => + { + return "nvarchar(max)"; + }); + // Массивы + TypeConverters.Add(typeof(Array), (sourceName, sourceType) => + { + return "nvarchar(max)"; + }); + // Словари + TypeConverters.Add(typeof(DictionaryBase), (sourceName, sourceType) => + { + return "nvarchar(max)"; + }); + // Кортежи + TypeConverters.Add(typeof(Tuple), (sourceName, sourceType) => + { + return "nvarchar(max)"; + }); + // Уникальный идентификатор + TypeConverters.Add(typeof(Guid), (sourceName, sourceType) => + { + return "uniqueidentifier"; + }); + } + + /// + /// Функция для выполнения запроса к ClickHouse и получения скалярного значения + /// + /// Строка подключения к ClickHouse + /// SQL-текст запроса для выполнения + /// Результат запроса, представленный строкой + [SqlFunction(DataAccess = DataAccessKind.Read)] + public static SqlChars ExecuteScalar(SqlChars connectionString, SqlChars queryText) + { + string connectionStringValue = new string(connectionString.Value); + string queryTextValue = new string(queryText.Value); + + string resultAsString = string.Empty; + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + var queryResult = connection.ExecuteScalarAsync(queryTextValue) + .GetAwaiter().GetResult(); + + resultAsString = queryResult.ToString(); + } + + return new SqlChars(resultAsString); + } + + + /// + /// Функция для выполнения простого запроса. + /// + /// Возвращается только первая колонка из результата запроса в виде строки. + /// + /// Строка подключения к ClickHouse + /// SQL-текст запроса для выполнения + /// Набор результата запроса (только первая колонка в виде строки) + [SqlFunction( + FillRowMethodName = "ExecuteSimpleFillRow", + SystemDataAccess = SystemDataAccessKind.Read, + DataAccess = DataAccessKind.Read)] + public static IEnumerable ExecuteSimple(SqlChars connectionString, SqlChars queryText) + { + List resultRows = new List(); + + string connectionStringValue = new string(connectionString.Value); + string queryTextValue = new string(queryText.Value); + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + using (var reader = connection.ExecuteReaderAsync(queryTextValue) + .GetAwaiter().GetResult()) + { + while(reader.Read()) + { + resultRows.Add(new ExecuteSimpleRowResult() + { + ResultValue = ConvertTypeToSqlCommandType(reader.GetValue(0)).ToString() + }); + } + } + } + + return resultRows; + } + public static void ExecuteSimpleFillRow(object resultRow, out SqlChars rowValue) + { + var resultRowObject = (ExecuteSimpleRowResult)resultRow; + rowValue = new SqlChars(resultRowObject.ResultValue); + } + + /// + /// Выполнение команды к ClickHouse без получения результата + /// + /// Строка подключения к ClickHouse + /// SQL-текст команды + [SqlProcedure] + public static void ExecuteStatement(SqlChars connectionString, SqlChars queryText) + { + string connectionStringValue = new string(connectionString.Value); + string queryTextValue = new string(queryText.Value); + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + var queryResult = connection.ExecuteStatementAsync(queryTextValue) + .GetAwaiter().GetResult(); + } + } + + /// + /// Функция для получения текста запроса создания временной таблицы + /// для сохранения результата запроса к ClickHouse + /// + /// Строка подключения к ClickHouse + /// SQL-текст запроса для выполнения + /// Текст SQL-запроса для создания временной таблицы результата запроса + [SqlFunction(DataAccess = DataAccessKind.Read)] + public static SqlChars GetCreateTempDbTableCommand(SqlChars connectionString, SqlChars queryText, SqlChars tempTableName) + { + string connectionStringValue = new string(connectionString.Value); + string queryTextValue = new string(queryText.Value); + // Устанавливаем LIMIT 0, чтобы запрос не возвращал результата. + // Используется только для анализа схемы данных. + string regexLimitStmt = @"limit[ ][\d]+"; + if (Regex.IsMatch(queryTextValue, regexLimitStmt, RegexOptions.IgnoreCase)) + { + queryTextValue = Regex.Replace(queryTextValue, regexLimitStmt, "LIMIT 0", RegexOptions.IgnoreCase); + } else + { + queryTextValue = queryTextValue + "\n LIMIT 0"; + } + + string tempTableNameValue = new string(tempTableName.Value); + if (!tempTableNameValue.StartsWith("#", StringComparison.InvariantCultureIgnoreCase)) + { + throw new Exception("Temp table name should begin with # (local temp table) or ## (global temp table)"); + } + + string resultAsString; + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + using (var reader = connection.ExecuteReaderAsync(queryTextValue) + .GetAwaiter().GetResult()) + { + // Анализ результата запроса и создание под него временной таблицы + StringBuilder queryCreateTempTable = new StringBuilder(); + queryCreateTempTable.Append("CREATE TABLE "); + queryCreateTempTable.Append(tempTableNameValue); + queryCreateTempTable.Append(" (\n"); + for (int i = 0; i < reader.FieldCount; i++) + { + string fieldName = reader.GetName(i); + Type fieldType = reader.GetFieldType(i); + int fieldNumber = i + 1; + + queryCreateTempTable.Append(" ["); + queryCreateTempTable.Append(fieldName); + queryCreateTempTable.Append("] "); + queryCreateTempTable.Append(ConvertClickHouseTypeToSqlType(fieldType, fieldName)); + + if (fieldNumber != reader.FieldCount) + { + queryCreateTempTable.Append(","); + } + + queryCreateTempTable.Append("\n"); + } + queryCreateTempTable.Append(")"); + + resultAsString = queryCreateTempTable.ToString(); + } + } + + return new SqlChars(resultAsString); + } + + /// + /// Выполнение запроса к ClickHouse с сохранением результата во временную локальную таблицу + /// + /// Строка подключения к ClickHouse + /// SQL-текст команды + /// Имя временной таблицы для сохранения результата + [SqlProcedure] + public static void ExecuteToTempTable(SqlChars connectionString, SqlChars queryText, SqlChars tempTableName) + { + string tempTableNameValue = new string(tempTableName.Value); + + if(!tempTableNameValue.StartsWith("#", StringComparison.InvariantCultureIgnoreCase)) + { + throw new Exception("Temp table name should begin with # (local temp table)"); + } + if (tempTableNameValue.StartsWith("##", StringComparison.InvariantCultureIgnoreCase)) + { + throw new Exception("Temp table name should begin with # (local temp table). Global temp table with ## not supported by this method."); + } + + ExecuteToTempTableInternal(connectionString, queryText, tempTableName, _emptyString); + } + + /// + /// Выполнение запроса к ClickHouse с сохранением результата во временную глобальную таблицу + /// + /// Строка подключения к ClickHouse + /// SQL-текст команды + /// Имя временной таблицы для сохранения результата + /// Строка подключения к SQL Server для выполнения BULK INSERT. + /// Если передана пустая строка, то вставка во временную таблицу будет выполняться обычными инструкциями INSERT. + /// + [SqlProcedure] + public static void ExecuteToGlobalTempTable(SqlChars connectionString, SqlChars queryText, SqlChars tempTableName, SqlChars sqlServerConnectionString) + { + string tempTableNameValue = new string(tempTableName.Value); + + if (!tempTableNameValue.StartsWith("##", StringComparison.InvariantCultureIgnoreCase)) + { + throw new Exception("Temp table name should begin with ## (global temp table)."); + } + + ExecuteToTempTableInternal(connectionString, queryText, tempTableName, sqlServerConnectionString); + } + + /// + /// Операция массовой вставки данных из временной таблицы SQL Server + /// в таблицу ClickHouse + /// + /// Строка подключения к ClickHouse + /// Имя временной таблицы с исходными данными + /// Имя таблицы ClickHouse для вставки данных + [SqlProcedure] + public static void ExecuteBulkInsertFromTempTable(SqlChars connectionString, SqlChars sourceTempTableName, SqlChars destinationTableName) + { + string connectionStringValue = new string(connectionString.Value); + string sourceTempTableNameValue = new string(sourceTempTableName.Value); + string destinationTableNameValue = new string(destinationTableName.Value); + + + using (SqlConnection sqlConnection = GetSqlConnection()) + { + if (sqlConnection.State != ConnectionState.Open) + { + sqlConnection.Open(); + } + + List rowsForInsert = new List(); + + string tempTableSelectQuery = + @" +SELECT * FROM " + sourceTempTableNameValue + @" +"; + using (SqlCommand tempTableReader = new SqlCommand(tempTableSelectQuery, sqlConnection)) + { + tempTableReader.CommandType = CommandType.Text; + + using (var reader = tempTableReader.ExecuteReader()) + { + while (reader.Read()) + { + object[] rowValues = new object[reader.FieldCount]; + for (int i = 0; i < reader.FieldCount; i++) + { + var columnValue = reader.GetValue(i); + rowValues[i] = columnValue; + } + rowsForInsert.Add(rowValues); + } + } + } + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + using (var bulkCopy = new ClickHouseBulkCopy(connection) + { + DestinationTableName = destinationTableNameValue, + BatchSize = 100000, + MaxDegreeOfParallelism = 1 + }) + { + bulkCopy.InitAsync().GetAwaiter().GetResult(); + bulkCopy.WriteToServerAsync(rowsForInsert).GetAwaiter().GetResult(); + } + } + } + } + + private static SqlConnection GetSqlConnection() + { + if (DebugConnection == null) + { + return new SqlConnection(ConnectionString); + } + + return DebugConnection; + } + private static bool SQLServerBulkInsertAvailable(DbDataReader reader, string destinationTableName, string sqlServerConnectionString = "") + { + // Должна быть заполнена строка подключения к SQL Server для BULK-операций, т.к. контекстное подключение + // не позволяет их выполнять + if (string.IsNullOrEmpty(sqlServerConnectionString)) + { + return false; + } + + // Проверка допустимых типов + bool arrayDetected = false; + bool dictionaryDetected = false; + bool tupleDetected = false; + for (int i = 0; i < reader.FieldCount; i++) + { + var fieldType = reader.GetFieldType(i); + + arrayDetected = arrayDetected || fieldType.IsArray; + dictionaryDetected = dictionaryDetected || IsDictionary(fieldType); + tupleDetected = tupleDetected || IsTupleType(fieldType); + } + if (arrayDetected || dictionaryDetected || tupleDetected) + { + return false; + } + + // BULK-операции доступны только для глобальных временных таблиц. + if (!destinationTableName.StartsWith("##")) + { + return false; + } + + return true; + } + private static void ExecuteToTempTableInternal(SqlChars connectionString, SqlChars queryText, SqlChars tempTableName, SqlChars sqlServerConnectionString) + { + string connectionStringValue = new string(connectionString.Value); + string queryTextValue = new string(queryText.Value); + string tempTableNameValue = new string(tempTableName.Value); + string sqlServerConnectionStringValue = new string(sqlServerConnectionString.Value); + + SqlConnection sqlConnection = GetSqlConnection(); + if (sqlConnection.State != ConnectionState.Open) + { + sqlConnection.Open(); + } + + // Проверяем наличие созданной временной таблицы + bool tempTableExists = false; + StringBuilder checkTempTableExists = new StringBuilder(); + checkTempTableExists.Append("SELECT\n"); + checkTempTableExists.Append(" CASE WHEN OBJECT_ID('tempdb.."); + checkTempTableExists.Append(tempTableNameValue); + checkTempTableExists.Append("') IS NULL\n"); + checkTempTableExists.Append(" THEN CAST(0 as bit)\n"); + checkTempTableExists.Append(" ELSE CAST(1 as bit)\n"); + checkTempTableExists.Append("END AS [TempTableExists]\n"); + using (SqlCommand command = new SqlCommand(checkTempTableExists.ToString(), sqlConnection)) + { + command.CommandType = CommandType.Text; + var checkResult = command.ExecuteScalar(); + tempTableExists = (bool)checkResult; + } + + // Создаем временную таблицу, если она не была создана ранее + if (!tempTableExists) + { + StringBuilder createTempTableIfNotExists = new StringBuilder(); + createTempTableIfNotExists.Append("IF(OBJECT_ID('tempdb.."); + createTempTableIfNotExists.Append(tempTableNameValue); + createTempTableIfNotExists.Append("') IS NULL)\n"); + createTempTableIfNotExists.AppendLine("BEGIN"); + createTempTableIfNotExists.Append( + new string(GetCreateTempDbTableCommand(connectionString, queryText, tempTableName).Value)); + createTempTableIfNotExists.AppendLine("END"); + using (SqlCommand command = new SqlCommand(createTempTableIfNotExists.ToString(), sqlConnection)) + { + command.CommandType = CommandType.Text; + command.ExecuteNonQuery(); + } + } + + using (var connection = new ClickHouseConnection(connectionStringValue)) + { + using (var reader = connection.ExecuteReaderAsync(queryTextValue) + .GetAwaiter().GetResult()) + { + if (SQLServerBulkInsertAvailable(reader, tempTableNameValue, sqlServerConnectionStringValue)) + { + WindowsImpersonationContext impersonatedIdentity = null; + if (SqlContext.IsAvailable) + { + WindowsIdentity currentIdentity = SqlContext.WindowsIdentity; + impersonatedIdentity = currentIdentity.Impersonate(); + } + + try + { + using (SqlConnection bulkInsertConnection = new SqlConnection(sqlServerConnectionStringValue)) + { + bulkInsertConnection.Open(); + using (SqlBulkCopy bc = new SqlBulkCopy(bulkInsertConnection)) + { + bc.DestinationTableName = tempTableNameValue; + bc.WriteToServer(reader); + } + bulkInsertConnection.Close(); + } + } + finally + { + if (impersonatedIdentity != null) + { + impersonatedIdentity.Undo(); + } + } + } + else + { + #region InsertTempDbTable + + StringBuilder queryInsertToTempTable = new StringBuilder(); + queryInsertToTempTable.Append("INSERT INTO "); + queryInsertToTempTable.Append(tempTableNameValue); + queryInsertToTempTable.Append(" VALUES ("); + for (int i = 0; i < reader.FieldCount; i++) + { + int fieldNumber = i + 1; + queryInsertToTempTable.Append("@P"); + queryInsertToTempTable.Append(fieldNumber); + if (fieldNumber != reader.FieldCount) + { + queryInsertToTempTable.Append(","); + } + + queryInsertToTempTable.Append("\n"); + } + + queryInsertToTempTable.Append(")"); + + using (SqlCommand command = new SqlCommand(queryInsertToTempTable.ToString(), sqlConnection)) + { + command.CommandType = CommandType.Text; + + while (reader.Read()) + { + command.Parameters.Clear(); + for (int i = 0; i < reader.FieldCount; i++) + { + object fieldValue = reader.GetValue(i); + int fieldNumber = i + 1; + command.Parameters.AddWithValue($"@P{fieldNumber}", + ConvertTypeToSqlCommandType(fieldValue)); + } + + command.ExecuteNonQuery(); + } + } + + #endregion + } + } + } + } + private static string ConvertClickHouseTypeToSqlType(Type sourceType, string sourceName) + { + Type typeForSearch = sourceType; + if (sourceType.IsArray) + { + sourceType = typeof(Array); + } + else if (IsDictionary(sourceType)) + { + sourceType = typeof(DictionaryBase); + } + else if (IsTupleType(sourceType)) + { + sourceType = typeof(Tuple); + } + + if (TypeConverters.TryGetValue(sourceType, out var convertFunc)) + { + return convertFunc(sourceName, sourceType); + } + else + { + throw new InvalidCastException($"Can't find type converter from {sourceType.Name}"); + } + } + private static object ConvertTypeToSqlCommandType(object sourceValue) + { + object prepearedValue; + Type sourceType = sourceValue.GetType(); + + if (sourceType.IsArray) + { + prepearedValue = JsonConvert.SerializeObject(sourceValue); + } + else if (IsDictionary(sourceType)) + { + prepearedValue = JsonConvert.SerializeObject(sourceValue); + } + else if (IsTupleType(sourceType)) + { + prepearedValue = JsonConvert.SerializeObject(sourceValue); + } + else + { + if (sourceValue is UInt16 || sourceValue is UInt32 || sourceValue is Int16) + { + prepearedValue = Convert.ToInt32(sourceValue); + } + else if (sourceValue is UInt64) + { + prepearedValue = Convert.ToDecimal(sourceValue); + } + else if (sourceValue is byte) + { + prepearedValue = Convert.ToInt32(sourceValue); + } + else if (sourceValue is IPAddress) + { + prepearedValue = sourceValue.ToString(); + } + else if (sourceValue is string + || sourceValue is decimal + || sourceValue is DateTime + || sourceValue is Guid) + { + prepearedValue = sourceValue; + } + else if (sourceValue is DateTimeOffset offset) + { + prepearedValue = offset.UtcDateTime; + } + else + { + prepearedValue = JsonConvert.SerializeObject(sourceValue); + } + } + + return prepearedValue; + } + private static bool IsTupleType(Type type, bool checkBaseTypes = false) + { + if (type == null) + throw new ArgumentNullException(nameof(type)); + + if (type == typeof(Tuple)) + return true; + + while (type != null) + { + if (type.IsGenericType) + { + var genType = type.GetGenericTypeDefinition(); + if (genType == typeof(Tuple<>) + || genType == typeof(Tuple<,>) + || genType == typeof(Tuple<,,>) + || genType == typeof(Tuple<,,,>) + || genType == typeof(Tuple<,,,,>) + || genType == typeof(Tuple<,,,,,>) + || genType == typeof(Tuple<,,,,,,>) + || genType == typeof(Tuple<,,,,,,,>) + || genType == typeof(Tuple<,,,,,,,>)) + return true; + } + + if (!checkBaseTypes) + break; + + type = type.BaseType; + } + + return false; + } + private static bool IsDictionary(Type type) + { + return type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>); + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Extensions/StringExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Extensions/StringExtensions.cs new file mode 100644 index 0000000..c693572 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Extensions/StringExtensions.cs @@ -0,0 +1,16 @@ +using System.Data.SqlTypes; + +namespace YPermitin.SQLCLR.ClickHouseClient.Entry.Extensions +{ + public static class StringExtensions + { + public static SqlChars ToSqlChars(this string value) { + return new SqlChars(value); + } + + public static string ToStringFromSqlChars(this SqlChars sqlString) + { + return new string(sqlString.Value); + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Models/ExecuteSimpleRowResult.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Models/ExecuteSimpleRowResult.cs new file mode 100644 index 0000000..40d1840 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Models/ExecuteSimpleRowResult.cs @@ -0,0 +1,7 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Models +{ + public class ExecuteSimpleRowResult + { + public string ResultValue { get; set; } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Properties/AssemblyInfo.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..489d6f1 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// Общие сведения об этой сборке предоставляются следующим набором +// набора атрибутов. Измените значения этих атрибутов для изменения сведений, +// связанные со сборкой. +[assembly: AssemblyTitle("ClickHouseClient.Entry")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ClickHouseClient.Entry")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Установка значения False для параметра ComVisible делает типы в этой сборке невидимыми +// для компонентов COM. Если необходимо обратиться к типу в этой сборке через +// COM, задайте атрибуту ComVisible значение TRUE для этого типа. +[assembly: ComVisible(false)] + +// Следующий GUID служит для идентификации библиотеки типов, если этот проект будет видимым для COM +[assembly: Guid("ac5f49bf-4526-47a3-8034-75f84108cee5")] + +// Сведения о версии сборки состоят из указанных ниже четырех значений: +// +// Основной номер версии +// Дополнительный номер версии +// Номер сборки +// Редакция +// +// Можно задать все значения или принять номера сборки и редакции по умолчанию +// используя "*", как показано ниже: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/packages.config b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/packages.config new file mode 100644 index 0000000..91e87f7 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient.Entry/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Adapters/ClickHouseDataAdapter.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Adapters/ClickHouseDataAdapter.cs new file mode 100644 index 0000000..489c596 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Adapters/ClickHouseDataAdapter.cs @@ -0,0 +1,11 @@ +using System.Data.Common; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO.Adapters +{ + /// + /// Dummy adapter class - to maintain backward compatibility + /// + public class ClickHouseDataAdapter : DbDataAdapter + { + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseCommand.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseCommand.cs new file mode 100644 index 0000000..e2a3aa7 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseCommand.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Readers; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + public class ClickHouseCommand : DbCommand, IClickHouseCommand, IDisposable + { + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private readonly ClickHouseParameterCollection commandParameters = new ClickHouseParameterCollection(); + private Dictionary customSettings; + private ClickHouseConnection connection; + + public ClickHouseCommand() + { + } + + public ClickHouseCommand(ClickHouseConnection connection) + { + this.connection = connection; + } + + public override string CommandText { get; set; } + + public override int CommandTimeout { get; set; } + + public override CommandType CommandType { get; set; } + + public override bool DesignTimeVisible { get; set; } + + public override UpdateRowSource UpdatedRowSource { get; set; } + + /// + /// Gets or sets QueryId associated with command + /// After query execution, will be set by value provided by server + /// Value will be same if provided or a UUID generated by server if not + /// + public string QueryId { get; set; } + + public QueryStats QueryStats { get; private set; } + + /// + /// Gets collection of custom settings which will be passed as URL query string parameters. + /// + /// Not thread-safe. + public IDictionary CustomSettings => customSettings ??= new Dictionary(); + + protected override DbConnection DbConnection + { + get => connection; + set => connection = (ClickHouseConnection)value; + } + + protected override DbParameterCollection DbParameterCollection => commandParameters; + + protected override DbTransaction DbTransaction { get; set; } + + public override void Cancel() => cts.Cancel(); + + public override int ExecuteNonQuery() => ExecuteNonQueryAsync(cts.Token).GetAwaiter().GetResult(); + + public override async Task ExecuteNonQueryAsync(CancellationToken cancellationToken) + { + if (connection == null) + throw new InvalidOperationException("Connection is not set"); + + using var lcts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + using var response = await PostSqlQueryAsync(CommandText, lcts.Token).ConfigureAwait(false); +#if NET5_0_OR_GREATER + using var reader = new ExtendedBinaryReader(await response.Content.ReadAsStreamAsync(lcts.Token).ConfigureAwait(false)); +#else + using var reader = new ExtendedBinaryReader(await response.Content.ReadAsStreamAsync().ConfigureAwait(false)); +#endif + + return reader.PeekChar() != -1 ? reader.Read7BitEncodedInt() : 0; + } + + /// + /// Allows to return raw result from a query (with custom FORMAT) + /// + /// Cancellation token + /// ClickHouseRawResult object containing response stream + public async Task ExecuteRawResultAsync(CancellationToken cancellationToken) + { + if (connection == null) + throw new InvalidOperationException("Connection is not set"); + + using var lcts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + var response = await PostSqlQueryAsync(CommandText, lcts.Token).ConfigureAwait(false); + return new ClickHouseRawResult(response); + } + + public override object ExecuteScalar() => ExecuteScalarAsync(cts.Token).GetAwaiter().GetResult(); + + public override async Task ExecuteScalarAsync(CancellationToken cancellationToken) + { + using var lcts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + using var reader = await ExecuteDbDataReaderAsync(CommandBehavior.Default, lcts.Token).ConfigureAwait(false); + return reader.Read() ? reader.GetValue(0) : null; + } + + public override void Prepare() { /* ClickHouse has no notion of prepared statements */ } + + public new ClickHouseDbParameter CreateParameter() => new ClickHouseDbParameter(); + + protected override DbParameter CreateDbParameter() => CreateParameter(); + +#pragma warning disable CA2215 // Dispose methods should call base class dispose + protected override void Dispose(bool disposing) +#pragma warning restore CA2215 // Dispose methods should call base class dispose + { + if (disposing) + { + // Dispose token source but do not cancel + cts.Dispose(); + } + } + + protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) => ExecuteDbDataReaderAsync(behavior, cts.Token).GetAwaiter().GetResult(); + + protected override async Task ExecuteDbDataReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) + { + if (connection == null) + throw new InvalidOperationException("Connection is not set"); + + using var lcts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + var sqlBuilder = new StringBuilder(CommandText); + switch (behavior) + { + case CommandBehavior.SingleRow: + sqlBuilder.Append(" LIMIT 1"); + break; + case CommandBehavior.SchemaOnly: + sqlBuilder.Append(" LIMIT 0"); + break; + default: + break; + } + var result = await PostSqlQueryAsync(sqlBuilder.ToString(), lcts.Token).ConfigureAwait(false); + return ClickHouseDataReader.FromHttpResponse(result, connection.TypeSettings); + } + + private async Task PostSqlQueryAsync(string sqlQuery, CancellationToken token) + { + if (connection == null) + throw new InvalidOperationException("Connection not set"); + //using var activity = connection.StartActivity("PostSqlQueryAsync"); + + var uriBuilder = connection.CreateUriBuilder(); + await connection.EnsureOpenAsync().ConfigureAwait(false); // Preserve old behavior + + uriBuilder.QueryId = QueryId; + uriBuilder.CommandQueryStringParameters = customSettings; + + using var postMessage = connection.UseFormDataParameters + ? BuildHttpRequestMessageWithFormData( + sqlQuery: sqlQuery, + uriBuilder: uriBuilder) + : BuildHttpRequestMessageWithQueryParams( + sqlQuery: sqlQuery, + uriBuilder: uriBuilder); + + //activity.SetQuery(sqlQuery); + + var response = await connection.HttpClient + .SendAsync(postMessage, HttpCompletionOption.ResponseHeadersRead, token) + .ConfigureAwait(false); + + QueryId = ExtractQueryId(response); + QueryStats = ExtractQueryStats(response); + //activity.SetQueryStats(QueryStats); + return await ClickHouseConnection.HandleError(response, sqlQuery).ConfigureAwait(false); + } + + private HttpRequestMessage BuildHttpRequestMessageWithQueryParams(string sqlQuery, ClickHouseUriBuilder uriBuilder) + { + if (commandParameters != null) + { + sqlQuery = commandParameters.ReplacePlaceholders(sqlQuery); + foreach (ClickHouseDbParameter parameter in commandParameters) + { + uriBuilder.AddSqlQueryParameter( + parameter.ParameterName, + HttpParameterFormatter.Format(parameter, connection.TypeSettings)); + } + } + + var uri = uriBuilder.ToString(); + + var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); + + connection.AddDefaultHttpHeaders(postMessage.Headers); + HttpContent content = new StringContent(sqlQuery); + content.Headers.ContentType = new MediaTypeHeaderValue("text/sql"); + if (connection.UseCompression) + { + content = new CompressedContent(content, DecompressionMethods.GZip); + } + + postMessage.Content = content; + + return postMessage; + } + + private HttpRequestMessage BuildHttpRequestMessageWithFormData(string sqlQuery, ClickHouseUriBuilder uriBuilder) + { + var content = new MultipartFormDataContent(); + + if (commandParameters != null) + { + sqlQuery = commandParameters.ReplacePlaceholders(sqlQuery); + + foreach (ClickHouseDbParameter parameter in commandParameters) + { + content.Add( + content: new StringContent(HttpParameterFormatter.Format(parameter, connection.TypeSettings)), + name: $"param_{parameter.ParameterName}"); + } + } + + content.Add( + content: new StringContent(sqlQuery), + name: "query"); + + var uri = uriBuilder.ToString(); + + var postMessage = new HttpRequestMessage(HttpMethod.Post, uri); + + connection.AddDefaultHttpHeaders(postMessage.Headers); + + postMessage.Content = content; + + return postMessage; + } + + /* + private static readonly JsonSerializerOptions SummarySerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = new SnakeCaseNamingPolicy(), + NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowReadingFromString, + }; + */ + + private static QueryStats ExtractQueryStats(HttpResponseMessage response) + { + try + { + const string summaryHeader = "X-ClickHouse-Summary"; + if (response.Headers.Contains(summaryHeader)) + { + var value = response.Headers.GetValues(summaryHeader).FirstOrDefault(); + //var jsonDoc = JsonDocument.Parse(value); + //return JsonSerializer.Deserialize(value, SummarySerializerOptions); + var queryStats = JsonConvert.DeserializeObject(value); + return queryStats; + } + } + catch + { + } + return null; + } + + private static string ExtractQueryId(HttpResponseMessage response) + { + const string queryIdHeader = "X-ClickHouse-Query-Id"; + if (response.Headers.Contains(queryIdHeader)) + return response.Headers.GetValues(queryIdHeader).FirstOrDefault(); + else + return null; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnection.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnection.cs new file mode 100644 index 0000000..1d3df69 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnection.cs @@ -0,0 +1,428 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using YPermitin.SQLCLR.ClickHouseClient.Http; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +//using Microsoft.Extensions.Logging; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + public class ClickHouseConnection : DbConnection, IClickHouseConnection, ICloneable, IDisposable + { + private const string CustomSettingPrefix = "set_"; + + private readonly List disposables = new(); + private readonly string httpClientName; + private readonly ConcurrentDictionary customSettings = new ConcurrentDictionary(); + private volatile ConnectionState state = ConnectionState.Closed; // Not an autoproperty because of interface implementation + + // Values provided by constructor + private HttpClient providedHttpClient; + private IHttpClientFactory providedHttpClientFactory; + // Actually used value + private IHttpClientFactory httpClientFactory; + + private Version serverVersion; + private string serverTimezone; + + private string database = ClickHouseEnvironment.Database; + private string username = ClickHouseEnvironment.Username; + private string password = ClickHouseEnvironment.Password; + private string session; + private bool useServerTimezone; + private bool useCustomDecimals; + private TimeSpan timeout; + private Uri serverUri; + private Feature supportedFeatures; + + public ClickHouseConnection() + : this(string.Empty) + { + } + + public ClickHouseConnection(string connectionString) + { + ConnectionString = connectionString; + } + + /// + /// Initializes a new instance of the class using provided HttpClient. + /// Note that HttpClient must have AutomaticDecompression enabled if compression is not disabled in connection string + /// + /// Connection string + /// instance of HttpClient + public ClickHouseConnection(string connectionString, HttpClient httpClient) + { + providedHttpClient = httpClient; + ConnectionString = connectionString; + } + + /// + /// Initializes a new instance of the class using an HttpClient generated by the provided . + /// + /// The ClickHouse connection string. + /// The factory to be used for creating the clients. + /// + /// The name of the HTTP client you want to be created using the provided factory. + /// If left empty, the default client will be created. + /// + /// + /// + /// + /// If compression is not disabled in the , the + /// must be configured to enable for its generated clients. + /// + /// For example you can do this while registering the HTTP client: + /// + /// services.AddHttpClient("ClickHouseClient").ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + /// { + /// AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + /// }); + /// + /// + /// + /// + /// The must set the timeout for its clients if needed. + /// + /// For example you can do this while registering the HTTP client: + /// + /// services.AddHttpClient("ClickHouseClient", c => c.Timeout = TimeSpan.FromMinutes(5)); + /// + /// + /// + /// + /// + public ClickHouseConnection(string connectionString, IHttpClientFactory httpClientFactory, string httpClientName = "") + { + this.providedHttpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + this.httpClientName = httpClientName ?? throw new ArgumentNullException(nameof(httpClientName)); + ConnectionString = connectionString; + } + + //public ILogger Logger { get; set; } + + /// + /// Gets or sets string defining connection settings for ClickHouse server + /// Example: Host=localhost;Port=8123;Username=default;Password=123;Compression=true + /// + public sealed override string ConnectionString + { + get => ConnectionStringBuilder.ToString(); + set => ConnectionStringBuilder = new ClickHouseConnectionStringBuilder() { ConnectionString = value }; + } + + public IDictionary CustomSettings => customSettings; + + public override ConnectionState State => state; + + public override string Database => database; + + internal string Username => username; + + internal Uri ServerUri => serverUri; + + internal string RedactedConnectionString + { + get + { + var builder = ConnectionStringBuilder; + builder.Password = "****"; + return builder.ToString(); + } + } + + public string ServerTimezone => serverTimezone; + + public override string DataSource { get; } + + public override string ServerVersion => serverVersion?.ToString(); + + public bool UseCompression { get; private set; } + + public bool UseFormDataParameters { get; private set; } + + public void SetFormDataParameters( + bool sendParametersAsFormData) + { + this.UseFormDataParameters = sendParametersAsFormData; + } + + /// + /// Gets enum describing which ClickHouse features are available on this particular server version + /// Requires connection to be in Open state + /// + public virtual Feature SupportedFeatures + { + get => state == ConnectionState.Open ? supportedFeatures : throw new InvalidOperationException(); + private set => supportedFeatures = value; + } + + private void ResetHttpClientFactory() + { + // If current httpClientFactory is owned by this connection, dispose of it + if (httpClientFactory is IDisposable d && disposables.Contains(d)) + { + d.Dispose(); + disposables.Remove(d); + } + + // If we have a HttpClient provided, use it + if (providedHttpClient != null) + { + httpClientFactory = new CannedHttpClientFactory(providedHttpClient); + } + + // If we have a provided client factory, use that + else if (providedHttpClientFactory != null) + { + httpClientFactory = providedHttpClientFactory; + } + + // If sessions are enabled, always use single connection + else if (!string.IsNullOrEmpty(session)) + { + var factory = new SingleConnectionHttpClientFactory() { Timeout = timeout }; + disposables.Add(factory); + httpClientFactory = factory; + } + + // Default case - use default connection pool + else + { + httpClientFactory = new DefaultPoolHttpClientFactory() { Timeout = timeout }; + } + } + + public override DataTable GetSchema() => GetSchema(null, null); + + public override DataTable GetSchema(string collectionName) => GetSchema(collectionName, null); + + public override DataTable GetSchema(string collectionName, string[] restrictionValues) => SchemaDescriber.DescribeSchema(this, collectionName, restrictionValues); + + internal static async Task HandleError(HttpResponseMessage response, string query) + { + if (response.IsSuccessStatusCode) + { + //activity.SetSuccess(); + return response; + } + var error = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var ex = ClickHouseServerException.FromServerResponse(error, query); + //activity.SetException(ex); + throw ex; + } + + public override void ChangeDatabase(string databaseName) => database = databaseName; + + public object Clone() => new ClickHouseConnection(ConnectionString); + + public override void Close() => state = ConnectionState.Closed; + + public override void Open() => OpenAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + + public override async Task OpenAsync(CancellationToken cancellationToken) + { + const string versionQuery = "SELECT version(), timezone() FORMAT TSV"; + + if (State == ConnectionState.Open) + return; + //using var activity = this.StartActivity("OpenAsync"); + //activity.SetQuery(versionQuery); + + try + { + var uriBuilder = CreateUriBuilder(); + var request = new HttpRequestMessage(HttpMethod.Post, uriBuilder.ToString()) + { + Content = new StringContent(versionQuery, Encoding.UTF8), + }; + AddDefaultHttpHeaders(request.Headers); + var response = await HandleError(await HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false), versionQuery).ConfigureAwait(false); +#if NET5_0_OR_GREATER + var data = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); +#else + var data = await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false); +#endif + + if (data.Length > 2 && data[0] == 0x1F && data[1] == 0x8B) // Check if response starts with GZip marker + throw new InvalidOperationException("ClickHouse server returned compressed result but HttpClient did not decompress it. Check HttpClient settings"); + + if (data.Length == 0) + throw new InvalidOperationException("ClickHouse server did not return version, check if the server is functional"); + + var serverVersionAndTimezone = Encoding.UTF8.GetString(data).Trim().Split('\t'); + + serverVersion = ParseVersion(serverVersionAndTimezone[0]); + serverTimezone = serverVersionAndTimezone[1]; + SupportedFeatures = ClickHouseFeatureMap.GetFeatureFlags(serverVersion); + state = ConnectionState.Open; + } + catch (Exception) + { + state = ConnectionState.Broken; + throw; + } + } + + /// + /// Warning: implementation-specific API. Exposed to allow custom optimizations + /// May change in future versions + /// + /// SQL query to add to URL, may be empty + /// Raw stream to be sent. May contain SQL query at the beginning. May be gzip-compressed + /// indicates whether "Content-Encoding: gzip" header should be added + /// Cancellation token + /// Task-wrapped HttpResponseMessage object + public async Task PostStreamAsync(string sql, Stream data, bool isCompressed, CancellationToken token) + { + var content = new StreamContent(data); + await PostStreamAsync(sql, content, isCompressed, token).ConfigureAwait(false); + } + + /// + /// Warning: implementation-specific API. Exposed to allow custom optimizations + /// May change in future versions + /// + /// SQL query to add to URL, may be empty + /// Callback invoked to write to the stream. May contain SQL query at the beginning. May be gzip-compressed + /// indicates whether "Content-Encoding: gzip" header should be added + /// Cancellation token + /// Task-wrapped HttpResponseMessage object + public async Task PostStreamAsync(string sql, Func callback, bool isCompressed, CancellationToken token) + { + var content = new StreamCallbackContent(callback, token); + await PostStreamAsync(sql, content, isCompressed, token).ConfigureAwait(false); + } + + private async Task PostStreamAsync(string sql, HttpContent content, bool isCompressed, CancellationToken token) + { + //using var activity = this.StartActivity("PostStreamAsync"); + //activity.SetQuery(sql); + + var builder = CreateUriBuilder(sql); + using var postMessage = new HttpRequestMessage(HttpMethod.Post, builder.ToString()); + AddDefaultHttpHeaders(postMessage.Headers); + + postMessage.Content = content; + postMessage.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + if (isCompressed) + { + postMessage.Content.Headers.Add("Content-Encoding", "gzip"); + } + using var response = await HttpClient.SendAsync(postMessage, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false); + await HandleError(response, sql).ConfigureAwait(false); + } + + public new ClickHouseCommand CreateCommand() => new ClickHouseCommand(this); + + void IDisposable.Dispose() + { + GC.SuppressFinalize(this); + foreach (var d in disposables) + d.Dispose(); + } + + internal static Version ParseVersion(string versionString) + { + if (string.IsNullOrWhiteSpace(versionString)) + throw new ArgumentException($"'{nameof(versionString)}' cannot be null or whitespace.", nameof(versionString)); + var parts = versionString.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries) + .Select(s => int.TryParse(s, NumberStyles.Integer, CultureInfo.InvariantCulture, out var i) ? i : 0) + .ToArray(); + if (parts.Length == 0 || parts[0] == 0) + throw new InvalidOperationException($"Invalid version: {versionString}"); + return new Version(parts.ElementAtOrDefault(0), parts.ElementAtOrDefault(1), parts.ElementAtOrDefault(2), parts.ElementAtOrDefault(3)); + } + + internal HttpClient HttpClient => httpClientFactory.CreateClient(httpClientName); + + internal TypeSettings TypeSettings => new TypeSettings(useCustomDecimals, useServerTimezone ? serverTimezone : TypeSettings.DefaultTimezone); + + internal ClickHouseUriBuilder CreateUriBuilder(string sql = null) => new ClickHouseUriBuilder(serverUri) + { + Database = database, + SessionId = session, + UseCompression = UseCompression, + ConnectionQueryStringParameters = customSettings + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + Sql = sql, + }; + + internal Task EnsureOpenAsync() => state != ConnectionState.Open ? OpenAsync() : Task.CompletedTask; + + internal void AddDefaultHttpHeaders(HttpRequestHeaders headers) + { + headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}"))); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/csv")); + headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/octet-stream")); + if (UseCompression) + { + headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); + headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("deflate")); + } + } + + internal ClickHouseConnectionStringBuilder ConnectionStringBuilder + { + get + { + var builder = new ClickHouseConnectionStringBuilder + { + Database = database, + Username = username, + Password = password, + Protocol = serverUri?.Scheme, + Host = serverUri?.Host, + Port = (ushort)serverUri?.Port, + Compression = UseCompression, + UseSession = session != null, + Timeout = timeout, + UseServerTimezone = useServerTimezone, + UseCustomDecimals = useCustomDecimals, + }; + + foreach (var kvp in CustomSettings) + builder[CustomSettingPrefix + kvp.Key] = kvp.Value; + + return builder; + } + + set + { + var builder = value; + database = builder.Database; + username = builder.Username; + password = builder.Password; + serverUri = new UriBuilder(builder.Protocol, builder.Host, builder.Port).Uri; + UseCompression = builder.Compression; + session = builder.UseSession ? builder.SessionId ?? Guid.NewGuid().ToString() : null; + timeout = builder.Timeout; + useServerTimezone = builder.UseServerTimezone; + useCustomDecimals = builder.UseCustomDecimals; + + foreach (var key in builder.Keys.Cast().Where(k => k.StartsWith(CustomSettingPrefix, true, CultureInfo.InvariantCulture))) + { + CustomSettings.Set(key.Replace(CustomSettingPrefix, string.Empty), builder[key]); + } + + ResetHttpClientFactory(); + } + } + + protected override DbTransaction BeginDbTransaction(IsolationLevel isolationLevel) => throw new NotSupportedException(); + + protected override DbCommand CreateDbCommand() => CreateCommand(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionFactory.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionFactory.cs new file mode 100644 index 0000000..b8e9307 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionFactory.cs @@ -0,0 +1,25 @@ +using System.Data.Common; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Adapters; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + public class ClickHouseConnectionFactory : DbProviderFactory + { + public static ClickHouseConnectionFactory Instance => new(); + + public override DbConnection CreateConnection() => new ClickHouseConnection(); + + public override DbDataAdapter CreateDataAdapter() => new ClickHouseDataAdapter(); + + public override DbConnectionStringBuilder CreateConnectionStringBuilder() => new ClickHouseConnectionStringBuilder(); + + public override DbParameter CreateParameter() => new ClickHouseDbParameter(); + + public override DbCommand CreateCommand() => new ClickHouseCommand(); + +#if NET7_0_OR_GREATER + public override DbDataSource CreateDataSource(string connectionString) => new ClickHouseDataSource(connectionString); +#endif + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionStringBuilder.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionStringBuilder.cs new file mode 100644 index 0000000..e400f85 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseConnectionStringBuilder.cs @@ -0,0 +1,119 @@ +using System; +using System.Data.Common; +using System.Globalization; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + public class ClickHouseConnectionStringBuilder : DbConnectionStringBuilder + { + public ClickHouseConnectionStringBuilder() + { + } + + public ClickHouseConnectionStringBuilder(string connectionString) + { + ConnectionString = connectionString; + } + + public string Database + { + get => GetStringOrDefault("Database", ClickHouseEnvironment.Database); + set => this["Database"] = value; + } + + public string Username + { + get => GetStringOrDefault("Username", ClickHouseEnvironment.Username); + set => this["Username"] = value; + } + + public string Password + { + get => GetStringOrDefault("Password", ClickHouseEnvironment.Password); + set => this["Password"] = value; + } + + public string Protocol + { + get => GetStringOrDefault("Protocol", "http"); + set => this["Protocol"] = value; + } + + public string Host + { + get => GetStringOrDefault("Host", "localhost"); + set => this["Host"] = value; + } + + public bool Compression + { + get => GetBooleanOrDefault("Compression", true); + set => this["Compression"] = value; + } + + public bool UseSession + { + get => GetBooleanOrDefault("UseSession", false); + set => this["UseSession"] = value; + } + + public string SessionId + { + get => GetStringOrDefault("SessionId", null); + set => this["SessionId"] = value; + } + + public ushort Port + { + get => (ushort)GetIntOrDefault("Port", Protocol == "https" ? 8443 : 8123); + set => this["Port"] = value; + } + + public bool UseServerTimezone + { + get => GetBooleanOrDefault("UseServerTimezone", true); + set => this["UseServerTimezone"] = value; + } + + public bool UseCustomDecimals + { + get => GetBooleanOrDefault("UseCustomDecimals", true); + set => this["UseCustomDecimals"] = value; + } + + public TimeSpan Timeout + { + get + { + return TryGetValue("Timeout", out var value) && value is string @string && double.TryParse(@string, NumberStyles.Any, CultureInfo.InvariantCulture, out var timeout) + ? TimeSpan.FromSeconds(timeout) + : TimeSpan.FromMinutes(2); + } + set => this["Timeout"] = value.TotalSeconds; + } + + private bool GetBooleanOrDefault(string name, bool @default) + { + if (TryGetValue(name, out var value)) + return "true".Equals(value as string, StringComparison.OrdinalIgnoreCase); + else + return @default; + } + + private string GetStringOrDefault(string name, string @default) + { + if (TryGetValue(name, out var value)) + return (string)value; + else + return @default; + } + + private int GetIntOrDefault(string name, int @default) + { + if (TryGetValue(name, out object o) && o is string s && int.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out int @int)) + return @int; + else + return @default; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseDataSource.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseDataSource.cs new file mode 100644 index 0000000..9f8fa9b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseDataSource.cs @@ -0,0 +1,130 @@ +#if NET7_0_OR_GREATER +using System; +using System.Data.Common; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Client.ADO; + +public sealed class ClickHouseDataSource : DbDataSource, IClickHouseDataSource +{ + private readonly IHttpClientFactory httpClientFactory; + private readonly string httpClientName; + private readonly HttpClient httpClient; + private readonly bool disposeHttpClient; + + /// + /// Initializes a new instance of the class using provided HttpClient. + /// Note that HttpClient must have AutomaticDecompression enabled if compression is not disabled in connection string + /// + /// Connection string + /// instance of HttpClient + /// dispose of the passed-in instance of HttpClient + public ClickHouseDataSource(string connectionString, HttpClient httpClient = null, bool disposeHttpClient = true) + { + ConnectionString = connectionString; + this.httpClient = httpClient; + this.disposeHttpClient = disposeHttpClient; + } + + /// + /// Initializes a new instance of the class using an HttpClient generated by the provided . + /// + /// The ClickHouse connection string. + /// The factory to be used for creating the clients. + /// + /// The name of the HTTP client you want to be created using the provided factory. + /// If left empty, the default client will be created. + /// + /// + /// + /// + /// If compression is not disabled in the , the + /// must be configured to enable for its generated clients. + /// + /// For example, you can do this while registering the HTTP client: + /// + /// services.AddHttpClient("ClickHouseClient").ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + /// { + /// AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + /// }); + /// + /// + /// + /// + /// The must set the timeout for its clients if needed. + /// + /// For example, you can do this while registering the HTTP client: + /// + /// services.AddHttpClient("ClickHouseClient", c => c.Timeout = TimeSpan.FromMinutes(5)); + /// + /// + /// + /// + /// + public ClickHouseDataSource(string connectionString, IHttpClientFactory httpClientFactory, string httpClientName = "") + { + ArgumentNullException.ThrowIfNull(httpClientFactory); + ArgumentNullException.ThrowIfNull(httpClientName); + ConnectionString = connectionString; + this.httpClientFactory = httpClientFactory; + this.httpClientName = httpClientName; + } + + public override string ConnectionString + { + get; + } + + public ILogger Logger + { + get; + set; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (disposing && disposeHttpClient) + { + httpClient?.Dispose(); + } + } + + protected override DbConnection CreateDbConnection() + { + var cn = httpClientFactory != null + ? new ClickHouseConnection(ConnectionString, httpClientFactory, httpClientName) + : new ClickHouseConnection(ConnectionString, httpClient); + if (cn.Logger == null && Logger != null) + { + cn.Logger = Logger; + } + + return cn; + } + + public new ClickHouseConnection CreateConnection() => (ClickHouseConnection)CreateDbConnection(); + + IClickHouseConnection IClickHouseDataSource.CreateConnection() => CreateConnection(); + + public new ClickHouseConnection OpenConnection() => (ClickHouseConnection)OpenDbConnection(); + + IClickHouseConnection IClickHouseDataSource.OpenConnection() => OpenConnection(); + + public new async Task OpenConnectionAsync(CancellationToken cancellationToken) + { + var cn = await OpenDbConnectionAsync(cancellationToken).ConfigureAwait(false); + return (ClickHouseConnection)cn; + } + + async Task IClickHouseDataSource.OpenConnectionAsync(CancellationToken cancellationToken) + { + var cn = await OpenDbConnectionAsync(cancellationToken).ConfigureAwait(false); + return (IClickHouseConnection)cn; + } +} +#endif diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseEnvironment.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseEnvironment.cs new file mode 100644 index 0000000..6b23f2f --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/ClickHouseEnvironment.cs @@ -0,0 +1,13 @@ +using System; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + internal static class ClickHouseEnvironment + { + public static string Database => Environment.GetEnvironmentVariable("CLICKHOUSE_DB") ?? "default"; + + public static string Username => Environment.GetEnvironmentVariable("CLICKHOUSE_USER") ?? "default"; + + public static string Password => Environment.GetEnvironmentVariable("CLICKHOUSE_PASSWORD") ?? string.Empty; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Feature.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Feature.cs new file mode 100644 index 0000000..6f70765 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Feature.cs @@ -0,0 +1,44 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + [Flags] + public enum Feature + { + None = 0, // Special value + + [SinceVersion("21.4")] + UUIDParameters = 32, + + [SinceVersion("21.4")] + Map = 64, + + [SinceVersion("21.12")] + Bool = 128, + + [SinceVersion("21.9")] + Date32 = 256, + + [SinceVersion("21.9")] + WideTypes = 512, + + [Obsolete] + [SinceVersion("20.5")] + Geo = 1024, + + [SinceVersion("22.6")] + Stats = 2048, + + [SinceVersion("22.8")] + AsyncInsert = 8192, + + [SinceVersion("24.1")] + Variant = 16384, + + [SinceVersion("22.3")] + ParamsInMultipartFormData = 32768, + + All = ~None, // Special value + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseDbParameter.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseDbParameter.cs new file mode 100644 index 0000000..24fa1bd --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseDbParameter.cs @@ -0,0 +1,51 @@ +using System; +using System.Data; +using System.Data.Common; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters +{ + public class ClickHouseDbParameter : DbParameter + { + public override DbType DbType { get; set; } + + public string ClickHouseType { get; set; } + + public override ParameterDirection Direction { get => ParameterDirection.Input; set { } } + + public override bool IsNullable { get; set; } + + public override string ParameterName { get; set; } + + public override int Size { get; set; } + + public override string SourceColumn { get; set; } + + public override bool SourceColumnNullMapping { get; set; } + + public override object Value { get; set; } + + public override void ResetDbType() { } + + public override string ToString() => $"{ParameterName}:{Value}"; + + public string QueryForm + { + get + { + if (ClickHouseType != null) + return $"{{{ParameterName}:{ClickHouseType}}}"; + + if (Value is decimal d) + { + var parts = decimal.GetBits(d); + int scale = (parts[3] >> 16) & 0x7F; + return $"{{{ParameterName}:Decimal128({scale})}}"; + } + + var chType = TypeConverter.ToClickHouseType(Value?.GetType() ?? typeof(DBNull)).ToString(); + return $"{{{ParameterName}:{chType}}}"; + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseParameterCollection.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseParameterCollection.cs new file mode 100644 index 0000000..6e7fd8d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Parameters/ClickHouseParameterCollection.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters +{ + internal class ClickHouseParameterCollection : DbParameterCollection + { + private readonly List parameters = new(); + + public override int Count => parameters.Count; + + public override object SyncRoot { get; } + + public override int Add(object value) + { + parameters.Add((ClickHouseDbParameter)value); + return parameters.Count - 1; + } + + public override void AddRange(Array values) => parameters.AddRange(values.Cast()); + + public override void Clear() => parameters.Clear(); + + public override bool Contains(object value) => parameters.Contains(value as ClickHouseDbParameter); + + public override bool Contains(string value) => parameters.Any(p => p.ParameterName == value); + + public override void CopyTo(Array array, int index) + { + for (int i = 0; i < parameters.Count; i++) + { + array.SetValue(parameters[i].Value, index + i); + } + } + + public override IEnumerator GetEnumerator() => parameters.GetEnumerator(); + + public override int IndexOf(object value) => parameters.IndexOf(value as ClickHouseDbParameter); + + public override int IndexOf(string parameterName) => parameters.FindIndex(x => x.ParameterName == parameterName); + + public override void Insert(int index, object value) => parameters.Insert(index, (ClickHouseDbParameter)value); + + public override void Remove(object value) => parameters.Remove(value as ClickHouseDbParameter); + + public override void RemoveAt(int index) => parameters.RemoveAt(index); + + public override void RemoveAt(string parameterName) => parameters.RemoveAll(p => p.ParameterName == parameterName); + + protected override DbParameter GetParameter(int index) => parameters[index]; + + protected override DbParameter GetParameter(string parameterName) => parameters[IndexOf(parameterName)]; + + protected override void SetParameter(int index, DbParameter value) => parameters[index] = (ClickHouseDbParameter)value; + + protected override void SetParameter(string parameterName, DbParameter value) + { + var index = IndexOf(parameterName); + if (index < 0) + Add(value); + else + SetParameter(index, value); + } + + public override string ToString() => string.Join(";", parameters); + + internal string ReplacePlaceholders(string sqlQuery) + { + if (FeatureSwitch.DisableReplacingParameters || parameters.Count == 0) + return sqlQuery; + + var replacements = new Dictionary(); + // Using foreach+TryAdd as parameter collection can in theory contain duplicate names + foreach (var p in parameters) + replacements.TryAdd("@" + p.ParameterName, p.QueryForm); + + return sqlQuery.ReplaceMultipleWords(replacements); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/QueryStats.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/QueryStats.cs new file mode 100644 index 0000000..f64acba --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/QueryStats.cs @@ -0,0 +1,16 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + + public record QueryStats( + long ReadRows, + long ReadBytes, + long WrittenRows, + long WrittenBytes, + long TotalRowsToRead, + long ResultRows, + long ResultBytes, + long ElapsedNs); + +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseDataReader.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseDataReader.cs new file mode 100644 index 0000000..93dc49a --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseDataReader.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Types; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO.Readers +{ + public class ClickHouseDataReader : DbDataReader, IEnumerator, IEnumerable, IDataRecord + { + private const int BufferSize = 512 * 1024; + + private readonly HttpResponseMessage httpResponse; // Used to dispose at the end of reader + private readonly ExtendedBinaryReader reader; + + private ClickHouseDataReader(HttpResponseMessage httpResponse, ExtendedBinaryReader reader, string[] names, ClickHouseType[] types) + { + this.httpResponse = httpResponse ?? throw new ArgumentNullException(nameof(httpResponse)); + this.reader = reader ?? throw new ArgumentNullException(nameof(reader)); + RawTypes = types; + FieldNames = names; + CurrentRow = new object[FieldNames.Length]; + } + + internal static ClickHouseDataReader FromHttpResponse(HttpResponseMessage httpResponse, TypeSettings settings) + { + if (httpResponse is null) throw new ArgumentNullException(nameof(httpResponse)); + ExtendedBinaryReader reader = null; + try + { + var stream = new BufferedStream(httpResponse.Content.ReadAsStreamAsync().GetAwaiter().GetResult(), BufferSize); + reader = new ExtendedBinaryReader(stream); // will dispose of stream + var (names, types) = ReadHeaders(reader, settings); + return new ClickHouseDataReader(httpResponse, reader, names, types); + } + catch (Exception) + { + httpResponse?.Dispose(); + reader?.Dispose(); + throw; + } + } + + internal ClickHouseType GetEffectiveClickHouseType(int ordinal) + { + var type = RawTypes[ordinal]; + return type is NullableType nt ? nt.UnderlyingType : type; + } + + internal ClickHouseType GetClickHouseType(int ordinal) => RawTypes[ordinal]; + + public override object this[int ordinal] => GetValue(ordinal); + + public override object this[string name] => this[GetOrdinal(name)]; + + public override int Depth { get; } + + public override int FieldCount => RawTypes?.Length ?? throw new InvalidOperationException(); + + public override bool IsClosed => false; + + public sealed override bool HasRows => true; + + public override int RecordsAffected { get; } + + protected object[] CurrentRow { get; set; } + + protected string[] FieldNames { get; set; } + + private protected ClickHouseType[] RawTypes { get; set; } + + public override bool GetBoolean(int ordinal) => Convert.ToBoolean(GetValue(ordinal), CultureInfo.InvariantCulture); + + public override byte GetByte(int ordinal) => (byte)GetValue(ordinal); + + public override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) => throw new NotImplementedException(); + + public override char GetChar(int ordinal) => (char)GetValue(ordinal); + + public override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) => throw new NotImplementedException(); + + public override string GetDataTypeName(int ordinal) => GetClickHouseType(ordinal).ToString(); + + public override DateTime GetDateTime(int ordinal) => (DateTime)GetValue(ordinal); + + public virtual DateTimeOffset GetDateTimeOffset(int ordinal) => GetEffectiveClickHouseType(ordinal) is AbstractDateTimeType adt ? + adt.CoerceToDateTimeOffset(GetDateTime(ordinal)) : throw new InvalidCastException(); + + public override decimal GetDecimal(int ordinal) + { + var value = GetValue(ordinal); + return value is ClickHouseDecimal clickHouseDecimal ? clickHouseDecimal.ToDecimal(CultureInfo.InvariantCulture) : (decimal)value; + } + + public override double GetDouble(int ordinal) => (double)GetValue(ordinal); + + public override Type GetFieldType(int ordinal) + { + var rawType = RawTypes[ordinal]; + return rawType is NullableType nt ? nt.UnderlyingType.FrameworkType : rawType.FrameworkType; + } + + public override float GetFloat(int ordinal) => (float)GetValue(ordinal); + + public override Guid GetGuid(int ordinal) => (Guid)GetValue(ordinal); + + public override short GetInt16(int ordinal) => (short)GetValue(ordinal); + + public override int GetInt32(int ordinal) => (int)GetValue(ordinal); + + public override long GetInt64(int ordinal) => (long)GetValue(ordinal); + + public override string GetName(int ordinal) => FieldNames[ordinal]; + + public override int GetOrdinal(string name) + { + var index = Array.FindIndex(FieldNames, (fn) => fn == name); + if (index == -1) + { + throw new ArgumentException("Column does not exist", nameof(name)); + } + + return index; + } + + public override string GetString(int ordinal) => (string)GetValue(ordinal); + + public override object GetValue(int ordinal) => CurrentRow[ordinal]; + + public override int GetValues(object[] values) + { + if (CurrentRow == null) + { + throw new InvalidOperationException(); + } + + CurrentRow.CopyTo(values, 0); + return CurrentRow.Length; + } + + public override bool IsDBNull(int ordinal) + { + var value = GetValue(ordinal); + return value is DBNull || value is null; + } + + public override bool NextResult() => false; + + public override void Close() => Dispose(); + + public override T GetFieldValue(int ordinal) => (T)GetValue(ordinal); + + public override DataTable GetSchemaTable() => SchemaDescriber.DescribeSchema(this); + + public override Task NextResultAsync(CancellationToken cancellationToken) => Task.FromResult(false); + + // Custom extension + public ushort GetUInt16(int ordinal) => (ushort)GetValue(ordinal); + + // Custom extension + public uint GetUInt32(int ordinal) => (uint)GetValue(ordinal); + + // Custom extension + public ulong GetUInt64(int ordinal) => (ulong)GetValue(ordinal); + + // Custom extension + public IPAddress GetIPAddress(int ordinal) => (IPAddress)GetValue(ordinal); + +#if !NET462 + // Custom extension + public ITuple GetTuple(int ordinal) => (ITuple)GetValue(ordinal); +#endif + + // Custom extension + public sbyte GetSByte(int ordinal) => (sbyte)GetValue(ordinal); + + // Custom extension + public BigInteger GetBigInteger(int ordinal) => (BigInteger)GetValue(ordinal); + + public override bool Read() + { + if (reader.PeekChar() == -1) + return false; // End of stream reached + + var count = RawTypes.Length; + var data = CurrentRow; + for (var i = 0; i < count; i++) + { + var rawType = RawTypes[i]; + data[i] = rawType.Read(reader); + } + return true; + } + +#pragma warning disable CA2215 // Dispose methods should call base class dispose + protected override void Dispose(bool disposing) + { + if (disposing) + { + httpResponse?.Dispose(); + reader?.Dispose(); + } + } +#pragma warning restore CA2215 // Dispose methods should call base class dispose + + private static (string[], ClickHouseType[]) ReadHeaders(ExtendedBinaryReader reader, TypeSettings settings) + { + if (reader.PeekChar() == -1) + { + return (new string[0], new ClickHouseType[0]); + } + var count = reader.Read7BitEncodedInt(); + var names = new string[count]; + var types = new ClickHouseType[count]; + + for (var i = 0; i < count; i++) + { + names[i] = reader.ReadString(); + } + + for (var i = 0; i < count; i++) + { + var chType = reader.ReadString(); + types[i] = TypeConverter.ParseClickHouseType(chType, settings); + } + return (names, types); + } + + public bool MoveNext() => Read(); + + public void Reset() => throw new NotSupportedException(); + + public override IEnumerator GetEnumerator() => this; + + IEnumerator IEnumerable.GetEnumerator() => this; + + public IDataReader Current => this; + + object IEnumerator.Current => this; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseRawResult.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseRawResult.cs new file mode 100644 index 0000000..af1e17a --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/Readers/ClickHouseRawResult.cs @@ -0,0 +1,31 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO.Readers +{ + public class ClickHouseRawResult : IDisposable + { + private readonly HttpResponseMessage response; + + internal ClickHouseRawResult(HttpResponseMessage response) + { + this.response = response; + } + + public Task ReadAsStreamAsync() => response.Content.ReadAsStreamAsync(); + + public Task ReadAsByteArrayAsync() => response.Content.ReadAsByteArrayAsync(); + + public Task ReadAsStringAsync() => response.Content.ReadAsStringAsync(); + + public Task CopyToAsync(Stream stream) => response.Content.CopyToAsync(stream); + + public void Dispose() + { + response?.Dispose(); + GC.SuppressFinalize(this); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/StreamCallbackContent.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/StreamCallbackContent.cs new file mode 100644 index 0000000..cbd5485 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ADO/StreamCallbackContent.cs @@ -0,0 +1,36 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace YPermitin.SQLCLR.ClickHouseClient.ADO +{ + /// + /// HttpContent implementation allowing streaming large payloads without having to materialize + /// the entire stream up-front. + /// + internal class StreamCallbackContent : HttpContent + { + private readonly Func callback; + private readonly CancellationToken cancellationToken; + + public StreamCallbackContent(Func callback, CancellationToken cancellationToken) + { + this.callback = callback; + this.cancellationToken = cancellationToken; + } + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) + { + return callback(stream, cancellationToken); + } + + protected override bool TryComputeLength(out long length) + { + length = 0; + return false; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseClient.csproj b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseClient.csproj new file mode 100644 index 0000000..6d28c13 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseClient.csproj @@ -0,0 +1,182 @@ + + + + + Debug + AnyCPU + {DE8E7D56-8A86-4142-84A7-5CEB3162329F} + Library + Properties + YPermitin.SQLCLR.ClickHouseClient + ClickHouseClient + v4.8 + 512 + true + 9.0 + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\packages\Newtonsoft.Json.8.0.3\lib\net45\Newtonsoft.Json.dll + + + ..\..\packages\NodaTime.1.4.7\lib\net35-Client\NodaTime.dll + + + + + + + + + + + + + + + Component + + + Component + + + Component + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseServerException.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseServerException.cs new file mode 100644 index 0000000..4e99ee2 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseServerException.cs @@ -0,0 +1,74 @@ +using System; +using System.Data.Common; +using System.Runtime.Serialization; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + /// + /// Exception class representing error which happened on ClickHouse server + /// + [Serializable] + public class ClickHouseServerException : DbException + { + public ClickHouseServerException() + { + } + + public ClickHouseServerException(string error, string query, int errorCode) + : base(error, errorCode) + { + Query = query; + } + + protected ClickHouseServerException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + + public string Query { get; } + + public static ClickHouseServerException FromServerResponse(string error, string query) + { + var errorCode = ParseErrorCode(error) ?? -1; + return new ClickHouseServerException(error, query, errorCode); + } + + private static int? ParseErrorCode(string error) + { + int start = -1; + int end = error.Length - 1; + + for (int i = 0; i < error.Length; i++) + { + if (char.IsDigit(error[i])) + { + start = i; + break; + } + } + + if (start == -1) + { + return null; + } + + for (int i = start; i < error.Length; i++) + { + if (!char.IsDigit(error[i])) + { + end = i; + break; + } + } + + if (int.TryParse(error.Substring(start, end - start), out int result)) + { + return result; + } + else + { + return null; + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseUriBuilder.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseUriBuilder.cs new file mode 100644 index 0000000..94be4b1 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/ClickHouseUriBuilder.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Web; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + + internal class ClickHouseUriBuilder + { + private readonly IDictionary sqlQueryParameters = new Dictionary(); + + public ClickHouseUriBuilder(Uri baseUri) + { + BaseUri = baseUri; + } + + public Uri BaseUri { get; } + + public string Sql { get; set; } + + public bool UseCompression { get; set; } + + public string Database { get; set; } + + public string SessionId { get; set; } + + public string QueryId { get; set; } + + public static string DefaultFormat => "RowBinaryWithNamesAndTypes"; + + public IDictionary ConnectionQueryStringParameters { get; set; } + + public IDictionary CommandQueryStringParameters { get; set; } + + public bool AddSqlQueryParameter(string name, string value) => + DictionaryExtensions.TryAdd(sqlQueryParameters, name, value); + + public override string ToString() + { + var parameters = HttpUtility.ParseQueryString(string.Empty); // NameValueCollection but a special one + parameters.Set( + "enable_http_compression", + UseCompression.ToString(CultureInfo.InvariantCulture).ToLowerInvariant()); + parameters.Set("default_format", DefaultFormat); + parameters.SetOrRemove("database", Database); + parameters.SetOrRemove("session_id", SessionId); + parameters.SetOrRemove("query", Sql); + parameters.SetOrRemove("query_id", QueryId); + + foreach (var parameter in sqlQueryParameters) + parameters.Set("param_" + parameter.Key, parameter.Value); + + if (ConnectionQueryStringParameters != null) + { + foreach (var parameter in ConnectionQueryStringParameters) + parameters.Set(parameter.Key, Convert.ToString(parameter.Value, CultureInfo.InvariantCulture)); + } + + if (CommandQueryStringParameters != null) + { + foreach (var parameter in CommandQueryStringParameters) + parameters.Set(parameter.Key, Convert.ToString(parameter.Value, CultureInfo.InvariantCulture)); + } + + var uriBuilder = new UriBuilder(BaseUri) { Query = parameters.ToString() }; + return uriBuilder.ToString(); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Constraints/DBDefault.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Constraints/DBDefault.cs new file mode 100644 index 0000000..ed91d39 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Constraints/DBDefault.cs @@ -0,0 +1,7 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Constraints +{ + public class DBDefault + { + public static readonly DBDefault Value = new(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Batch.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Batch.cs new file mode 100644 index 0000000..e157735 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Batch.cs @@ -0,0 +1,24 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Types; +//using System.Buffers; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy +{ + // Convenience argument collection + internal struct Batch : IDisposable + { + public object[][] Rows; + public int Size; + public string Query; + public ClickHouseType[] Types; + + public void Dispose() + { + if (Rows != null) + { + // ArrayPool.Shared.Return(Rows, true); + Rows = null; + } + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/ClickHouseBulkCopy.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/ClickHouseBulkCopy.cs new file mode 100644 index 0000000..d35a32d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/ClickHouseBulkCopy.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using YPermitin.SQLCLR.ClickHouseClient.ADO; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Readers; +using YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer; +using YPermitin.SQLCLR.ClickHouseClient.Types; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +//using Microsoft.IO; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy +{ + public class ClickHouseBulkCopy : IDisposable + { + //private static readonly RecyclableMemoryStreamManager MemoryStreamManager = new(); + private readonly ClickHouseConnection connection; + private readonly IBatchSerializer batchSerializer; + private readonly RowBinaryFormat rowBinaryFormat; + private readonly bool ownsConnection; + private long rowsWritten; + private (string[] names, ClickHouseType[] types) columnNamesAndTypes; + + public ClickHouseBulkCopy(ClickHouseConnection connection) : this(connection, RowBinaryFormat.RowBinary) { } + + public ClickHouseBulkCopy(string connectionString) : this(connectionString, RowBinaryFormat.RowBinary) { } + + public ClickHouseBulkCopy(ClickHouseConnection connection, RowBinaryFormat rowBinaryFormat) + { + this.connection = connection ?? throw new ArgumentNullException(nameof(connection)); + this.rowBinaryFormat = rowBinaryFormat; + batchSerializer = BatchSerializer.GetByRowBinaryFormat(rowBinaryFormat); + } + + public ClickHouseBulkCopy(string connectionString, RowBinaryFormat rowBinaryFormat) + : this( + string.IsNullOrWhiteSpace(connectionString) + ? throw new ArgumentNullException(nameof(connectionString)) + : new ClickHouseConnection(connectionString), + rowBinaryFormat) + { + ownsConnection = true; + } + + /// + /// Gets or sets size of batch in rows. + /// + public int BatchSize { get; set; } = 100000; + + /// + /// Gets or sets maximum number of parallel processing tasks. + /// + public int MaxDegreeOfParallelism { get; set; } = 4; + + /// + /// Gets name of destination table to insert to + /// + public string DestinationTableName { get; init; } + + /// + /// Gets columns + /// + public IReadOnlyCollection ColumnNames { get; init; } + + private async Task<(string[] names, ClickHouseType[] types)> LoadNamesAndTypesAsync(string destinationTableName, IReadOnlyCollection columns = null) + { + using var reader = (ClickHouseDataReader)await connection.ExecuteReaderAsync($"SELECT {GetColumnsExpression(columns)} FROM {DestinationTableName} WHERE 1=0").ConfigureAwait(false); + var types = reader.GetClickHouseColumnTypes(); + var names = reader.GetColumnNames().Select(c => c.EncloseColumnName()).ToArray(); + return (names, types); + } + + /// + /// Gets total number of rows written by this instance. + /// + public long RowsWritten => Interlocked.Read(ref rowsWritten); + + /// + /// One-time init operation to load column types using provided names + /// Required to call before WriteToServerAsync + /// + /// Awaitable task + public async Task InitAsync() + { + if (DestinationTableName is null) + throw new InvalidOperationException($"{nameof(DestinationTableName)} is null"); + columnNamesAndTypes = await LoadNamesAndTypesAsync(DestinationTableName, ColumnNames).ConfigureAwait(false); + } + + public Task WriteToServerAsync(IDataReader reader) => WriteToServerAsync(reader, CancellationToken.None); + + public Task WriteToServerAsync(IDataReader reader, CancellationToken token) + { + if (reader is null) + throw new ArgumentNullException(nameof(reader)); + + return WriteToServerAsync(reader.AsEnumerable(), token); + } + + public Task WriteToServerAsync(DataTable table, CancellationToken token) + { + if (table is null) + throw new ArgumentNullException(nameof(table)); + + var rows = table.Rows.Cast().Select(r => r.ItemArray); + return WriteToServerAsync(rows, token); + } + + public Task WriteToServerAsync(IEnumerable rows) => WriteToServerAsync(rows, CancellationToken.None); + + public async Task WriteToServerAsync(IEnumerable rows, CancellationToken token) + { + if (rows is null) + throw new ArgumentNullException(nameof(rows)); + + if (string.IsNullOrWhiteSpace(DestinationTableName)) + throw new InvalidOperationException("Destination table not set"); + + var (columnNames, columnTypes) = columnNamesAndTypes; + if (columnNames == null || columnTypes == null) + throw new InvalidOperationException("Column names not initialized. Call InitAsync once to load column data"); + + var query = $"INSERT INTO {DestinationTableName} ({string.Join(", ", columnNames)}) FORMAT {rowBinaryFormat.ToString()}"; + + var tasks = new Task[MaxDegreeOfParallelism]; + for (var i = 0; i < tasks.Length; i++) + { + tasks[i] = Task.CompletedTask; + } + + foreach (var batch in IntoBatches(rows, query, columnTypes)) + { + while (true) + { + var completedTaskIndex = Array.FindIndex(tasks, t => t.IsCompleted); + if (completedTaskIndex >= 0) + { + tasks[completedTaskIndex] = SendBatchAsync(batch, token); + break; // while (true); go to next batch + } + else + { + var completedTask = await Task.WhenAny(tasks).ConfigureAwait(false); + await completedTask.ConfigureAwait(false); + } + } + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + private async Task SendBatchAsync(Batch batch, CancellationToken token) + { + using (batch) // Dispose object regardless whether sending succeeds + { + //using var stream = MemoryStreamManager.GetStream(nameof(SendBatchAsync)); + using var stream = new MemoryStream(); + // Async serialization + await Task.Run(() => batchSerializer.Serialize(batch, stream), token).ConfigureAwait(false); + // Seek to beginning as after writing it's at end + stream.Seek(0, SeekOrigin.Begin); + // Async sending + await connection.PostStreamAsync(null, stream, true, token).ConfigureAwait(false); + // Increase counter + Interlocked.Add(ref rowsWritten, batch.Size); + } + } + + public void Dispose() + { + if (ownsConnection) + { + connection?.Dispose(); + } + GC.SuppressFinalize(this); + } + + private static string GetColumnsExpression(IReadOnlyCollection columns) => columns == null || columns.Count == 0 ? "*" : string.Join(",", columns); + + private IEnumerable IntoBatches(IEnumerable rows, string query, ClickHouseType[] types) + { + foreach (var (batch, size) in rows.BatchRented(BatchSize)) + { + yield return new Batch { Rows = batch, Size = size, Query = query, Types = types }; + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/RowBinaryFormat.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/RowBinaryFormat.cs new file mode 100644 index 0000000..b63b7a7 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/RowBinaryFormat.cs @@ -0,0 +1,8 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Copy +{ + public enum RowBinaryFormat + { + RowBinary, + RowBinaryWithDefaults, + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/BatchSerializer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/BatchSerializer.cs new file mode 100644 index 0000000..b269a6d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/BatchSerializer.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Text; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + internal class BatchSerializer : IBatchSerializer + { + public static BatchSerializer GetByRowBinaryFormat(RowBinaryFormat format) + { + return format switch + { + RowBinaryFormat.RowBinary => new BatchSerializer(new RowBinarySerializer()), + RowBinaryFormat.RowBinaryWithDefaults => new BatchSerializer(new RowBinaryWithDefaultsSerializer()), + _ => throw new NotSupportedException(format.ToString()) + }; + } + + private readonly IRowSerializer rowSerializer; + + public BatchSerializer(IRowSerializer rowSerializer) + { + this.rowSerializer = rowSerializer; + } + + public void Serialize(Batch batch, Stream stream) + { + using var gzipStream = new BufferedStream(new GZipStream(stream, CompressionLevel.Fastest, true), 256 * 1024); + using (var textWriter = new StreamWriter(gzipStream, Encoding.UTF8, 4 * 1024, true)) + { + textWriter.WriteLine(batch.Query); + } + + using var writer = new ExtendedBinaryWriter(gzipStream); + + object[] row = null; + int counter = 0; + var enumerator = batch.Rows.GetEnumerator(); + try + { + while (enumerator.MoveNext()) + { + row = (object[])enumerator.Current; + rowSerializer.Serialize(row, batch.Types, writer); + + counter++; + if (counter >= batch.Size) + break; // We've reached the batch size + } + } + catch (Exception e) + { + throw new ClickHouseBulkCopySerializationException(row, e); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/ClickHouseBulkCopySerializationException.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/ClickHouseBulkCopySerializationException.cs new file mode 100644 index 0000000..9d3b4f2 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/ClickHouseBulkCopySerializationException.cs @@ -0,0 +1,18 @@ +using System; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + public class ClickHouseBulkCopySerializationException : Exception + { + public ClickHouseBulkCopySerializationException(object[] row, Exception innerException) + : base("Error when serializing data", innerException) + { + Row = row; + } + + /// + /// Gets row at which exception happened + /// + public object[] Row { get; } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IBatchSerializer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IBatchSerializer.cs new file mode 100644 index 0000000..f1d8f10 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IBatchSerializer.cs @@ -0,0 +1,9 @@ +using System.IO; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + internal interface IBatchSerializer + { + void Serialize(Batch batch, Stream stream); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IRowSerializer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IRowSerializer.cs new file mode 100644 index 0000000..4d77708 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/IRowSerializer.cs @@ -0,0 +1,10 @@ +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + internal interface IRowSerializer + { + void Serialize(object[] row, ClickHouseType[] types, ExtendedBinaryWriter writer); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinarySerializer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinarySerializer.cs new file mode 100644 index 0000000..6f1e633 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinarySerializer.cs @@ -0,0 +1,16 @@ +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + internal class RowBinarySerializer : IRowSerializer + { + public void Serialize(object[] row, ClickHouseType[] types, ExtendedBinaryWriter writer) + { + for (int col = 0; col < row.Length; col++) + { + types[col].Write(writer, row[col]); + } + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinaryWithDefaultsSerializer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinaryWithDefaultsSerializer.cs new file mode 100644 index 0000000..79e88fa --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Copy/Serializer/RowBinaryWithDefaultsSerializer.cs @@ -0,0 +1,26 @@ +using YPermitin.SQLCLR.ClickHouseClient.Constraints; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.Copy.Serializer +{ + // https://clickhouse.com/docs/en/interfaces/formats#rowbinarywithdefaults + internal class RowBinaryWithDefaultsSerializer : IRowSerializer + { + public void Serialize(object[] row, ClickHouseType[] types, ExtendedBinaryWriter writer) + { + for (int col = 0; col < row.Length; col++) + { + if (row[col] is DBDefault) + { + writer.Write((byte)1); + } + else + { + writer.Write((byte)0); + types[col].Write(writer, row[col]); + } + } + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Diagnostic/ActivitySourceHelper.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Diagnostic/ActivitySourceHelper.cs new file mode 100644 index 0000000..6bd55d6 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Diagnostic/ActivitySourceHelper.cs @@ -0,0 +1,115 @@ +/* +using System; +using System.Diagnostics; +using System.Globalization; +using System.Reflection; +using ClickHouse.Client.ADO; + +namespace ClickHouse.Client.Diagnostic +{ + internal static class ActivitySourceHelper + { + internal const string ActivitySourceName = "ClickHouse.Client"; + + private const string TagDbConnectionString = "db.connection_string"; + private const string TagDbName = "db.name"; + private const string TagDbStatement = "db.statement"; + private const string TagDbSystem = "db.system"; + private const string TagStatusCode = "otel.status_code"; + private const string TagUser = "db.user"; + private const string TagService = "peer.service"; + private const string TagThreadId = "thread.id"; + private const string TagReadRows = "db.clickhouse.read_rows"; + private const string TagReadBytes = "db.clickhouse.read_bytes"; + private const string TagWrittenRows = "db.clickhouse.written_rows"; + private const string TagWrittenBytes = "db.clickhouse.written_bytes"; + private const string TagResultRows = "db.clickhouse.result_rows"; + private const string TagResultBytes = "db.clickhouse.result_bytes"; + private const string TagElapsedNs = "db.clickhouse.elapsed_ns"; + + internal const int StatementMaxLen = 300; + + internal static ActivitySource ActivitySource { get; } = CreateActivitySource(); + + internal static Activity StartActivity(this ClickHouseConnection connection, string name) + { + if (connection is null) throw new ArgumentNullException(nameof(connection)); + if (string.IsNullOrEmpty(name)) throw new ArgumentNullException(nameof(name)); + + var activity = ActivitySource.StartActivity(name, ActivityKind.Client, default(ActivityContext)); + + if (activity is null) + return null; + + if (activity.IsAllDataRequested) + { + activity.SetTag(TagThreadId, Environment.CurrentManagedThreadId.ToString(CultureInfo.InvariantCulture)); + activity.SetTag(TagDbSystem, "clickhouse"); + } + activity.SetTag(TagDbConnectionString, connection.RedactedConnectionString); + activity.SetTag(TagDbName, connection.Database); + activity.SetTag(TagUser, connection.Username); + activity.SetTag(TagService, $"{connection.ServerUri.Host}:{connection.ServerUri.Port}"); + return activity; + } + + internal static void SetQuery(this Activity activity, string sql) + { + if (activity is null || sql is null) + return; + if (sql.Length > StatementMaxLen) + { + sql = sql.Substring(0, StatementMaxLen); + } + activity.SetTag(TagDbStatement, sql); + } + + internal static void SetQueryStats(this Activity activity, QueryStats stats) + { + if (activity is null || stats is null) + return; + activity.SetTag(TagReadRows, stats.ReadRows); + activity.SetTag(TagReadBytes, stats.ReadBytes); + activity.SetTag(TagWrittenRows, stats.WrittenRows); + activity.SetTag(TagWrittenBytes, stats.WrittenBytes); + activity.SetTag(TagResultRows, stats.ResultRows); + activity.SetTag(TagResultBytes, stats.ResultBytes); + activity.SetTag(TagElapsedNs, stats.ElapsedNs); + } + + internal static void SetSuccess(this Activity activity) + { +#if NET6_0_OR_GREATER + activity?.SetStatus(ActivityStatusCode.Ok); +#endif + activity?.SetTag(TagStatusCode, "OK"); + activity?.Stop(); + } + + internal static void SetException(this Activity activity, Exception exception) + { + if (exception is null) throw new ArgumentNullException(nameof(exception)); + + var description = exception.Message; +#if NET6_0_OR_GREATER + activity?.SetStatus(ActivityStatusCode.Error, description); +#endif + activity?.SetTag(TagStatusCode, "ERROR"); + activity?.SetTag("otel.status_description", description); + activity?.AddEvent(new ActivityEvent("exception", tags: new ActivityTagsCollection + { + { "exception.type", exception?.GetType().FullName }, + { "exception.message", exception?.Message }, + })); + activity?.Stop(); + } + + private static ActivitySource CreateActivitySource() + { + var assembly = typeof(ActivitySourceHelper).Assembly; + var version = assembly.GetCustomAttribute().Version; + return new ActivitySource(ActivitySourceName, version); + } + } +} +*/ \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/FeatureSwitch.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/FeatureSwitch.cs new file mode 100644 index 0000000..5404386 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/FeatureSwitch.cs @@ -0,0 +1,24 @@ +using System; +using System.Linq; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + internal class FeatureSwitch + { + private const string Prefix = "ClickHouse.Client."; + + // Field names are used as switch + public static readonly bool DisableReplacingParameters; + + static FeatureSwitch() + { + var fields = typeof(FeatureSwitch).GetFields().Where(f => f.FieldType == typeof(bool)); + foreach (var field in fields) + { + var switchName = Prefix + field.Name; + AppContext.TryGetSwitch(switchName, out bool switchValue); + field.SetValue(null, switchValue); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryReader.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryReader.cs new file mode 100644 index 0000000..b7c6419 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryReader.cs @@ -0,0 +1,59 @@ +using System.IO; +using System.Text; + +namespace YPermitin.SQLCLR.ClickHouseClient.Formats +{ + + internal class ExtendedBinaryReader : BinaryReader + { + private readonly PeekableStreamWrapper streamWrapper; + + public ExtendedBinaryReader(Stream stream) + : base(new PeekableStreamWrapper(stream), Encoding.UTF8, false) + { + streamWrapper = (PeekableStreamWrapper)BaseStream; + } + + public new int Read7BitEncodedInt() => base.Read7BitEncodedInt(); + + /// + /// Performs guaranteed read of requested number of bytes, or throws an exception + /// + /// number of bytes to read + /// number of bytes read, always equals to count + /// thrown if requested number of bytes is not available + public override byte[] ReadBytes(int count) + { + var buffer = new byte[count]; + Read(buffer, 0, count); + return buffer; + } + + /// + /// Performs guaranteed read of requested number of bytes, or throws an exception + /// + /// buffer array + /// index to write to in the buffer + /// number of bytes to read + /// number of bytes read, always equals to count + /// thrown if requested number of bytes is not available + public override int Read(byte[] buffer, int index, int count) + { + int bytesRead = 0; + do + { + int read = base.Read(buffer, index + bytesRead, count - bytesRead); + bytesRead += read; + if (read == 0 && bytesRead < count) + { + throw new EndOfStreamException($"Expected to read {count} bytes, got {bytesRead}"); + } + } + while (bytesRead < count); + + return bytesRead; + } + + public override int PeekChar() => streamWrapper.Peek(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryWriter.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryWriter.cs new file mode 100644 index 0000000..d4f75dc --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/ExtendedBinaryWriter.cs @@ -0,0 +1,13 @@ +using System.IO; +using System.Text; + +namespace YPermitin.SQLCLR.ClickHouseClient.Formats +{ + public class ExtendedBinaryWriter : BinaryWriter + { + public ExtendedBinaryWriter(Stream stream) + : base(stream, Encoding.UTF8, false) { } + + public new void Write7BitEncodedInt(int i) => base.Write7BitEncodedInt(i); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/HttpParameterFormatter.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/HttpParameterFormatter.cs new file mode 100644 index 0000000..4fe76f4 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/HttpParameterFormatter.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters; +using YPermitin.SQLCLR.ClickHouseClient.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Types; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.Formats +{ + internal static class HttpParameterFormatter + { + private const string NullValueString = "\\N"; + + public static string Format(ClickHouseDbParameter parameter, TypeSettings settings) + { + var type = string.IsNullOrWhiteSpace(parameter.ClickHouseType) + ? TypeConverter.ToClickHouseType(parameter.Value.GetType()) + : TypeConverter.ParseClickHouseType(parameter.ClickHouseType, settings); + return Format(type, parameter.Value, false); + } + + internal static string Format(ClickHouseType type, object value, bool quote) + { + switch (type) + { + case NothingType nt: + return NullValueString; + case BooleanType bt: + return (bool)value ? "true" : "false"; + case IntegerType it: + case FloatType ft: + return Convert.ToString(value, CultureInfo.InvariantCulture); + case DecimalType dt when value is ClickHouseDecimal chd: + return chd.ToString(CultureInfo.InvariantCulture); + case DecimalType dt: + return Convert.ToDecimal(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + + case DateType dt when value is DateTimeOffset @dto: + return @dto.Date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); +#if NET6_0_OR_GREATER + case DateType dt when value is DateOnly @do: + return @do.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); +#endif + case DateType dt: + return Convert.ToDateTime(value, CultureInfo.InvariantCulture).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + + case StringType st: + case FixedStringType tt: + case Enum8Type e8t: + case Enum16Type e16t: + case IPv4Type ip4: + case IPv6Type ip6: + case UuidType uuidType: + return quote ? value.ToString().Escape().QuoteSingle() : value.ToString().Escape(); + + case LowCardinalityType lt: + return Format(lt.UnderlyingType, value, quote); + + case DateTimeType dtt when value is DateTime dt: + return dt.ToString("s", CultureInfo.InvariantCulture); + + case DateTimeType dtt when value is DateTimeOffset dto: + return dto.ToString("s", CultureInfo.InvariantCulture); + + case DateTime64Type dtt when value is DateTime dtv: + return $"{dtv:yyyy-MM-dd HH:mm:ss.fffffff}"; + + case DateTime64Type dtt when value is DateTimeOffset dto: + return $"{dto:yyyy-MM-dd HH:mm:ss.fffffff}"; + + case NullableType nt: + return value is null || value is DBNull ? quote ? "null" : NullValueString : Format(nt.UnderlyingType, value, quote); + + case ArrayType arrayType when value is IEnumerable enumerable: + return $"[{string.Join(",", enumerable.Cast().Select(obj => Format(arrayType.UnderlyingType, obj, true)))}]"; + + case NestedType nestedType when value is IEnumerable enumerable: + var values = enumerable.Cast().Select(x => Format(nestedType, x, false)); + return $"[{string.Join(",", values)}]"; + +#if !NET462 + case TupleType tupleType when value is ITuple tuple: + return $"({string.Join(",", tupleType.UnderlyingTypes.Select((x, i) => Format(x, tuple[i], true)))})"; +#endif + + case TupleType tupleType when value is IList list: + return $"({string.Join(",", tupleType.UnderlyingTypes.Select((x, i) => Format(x, list[i], true)))})"; + + case MapType mapType when value is IDictionary dict: + var strings = string.Join(",", dict.Keys.Cast().Select(k => $"{Format(mapType.KeyType, k, true)} : {Format(mapType.ValueType, dict[k], true)}")); + return $"{{{string.Join(",", strings)}}}"; + + case VariantType variantType: + var (_, chType) = variantType.GetMatchingType(value); + return Format(chType, value, quote); + + default: + throw new ArgumentException($"Cannot convert {value} to {type}"); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/PeekableStreamWrapper.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/PeekableStreamWrapper.cs new file mode 100644 index 0000000..260ef7b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Formats/PeekableStreamWrapper.cs @@ -0,0 +1,77 @@ +using System; +using System.IO; + +namespace YPermitin.SQLCLR.ClickHouseClient.Formats +{ + /// + /// Universal stream wrapper allowing to add 'PeekChar' support to streams with CanSeek=false + /// Suggestions for better solution wanted + /// + internal class PeekableStreamWrapper : Stream, IDisposable + { + private readonly Stream stream; + private bool hasReadAheadByte; + private int readAheadByte; + + public PeekableStreamWrapper(Stream stream) + { + this.stream = stream; + hasReadAheadByte = false; + readAheadByte = 0; + } + + public override bool CanRead => stream.CanRead; + + public override bool CanSeek => stream.CanSeek; + + public override bool CanWrite => stream.CanWrite; + + public override long Length => stream.Length; + + public override long Position { get => stream.Position; set => stream.Position = value; } + + public override void Flush() => stream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) + { + if (count == 0) + return 0; + var b = ReadByte(); + if (b == -1) + throw new EndOfStreamException(); + buffer[offset] = (byte)b; + var result = 1; + if (count > 1) + result += stream.Read(buffer, offset + 1, count - 1); + return result; + } + + public override long Seek(long offset, SeekOrigin origin) => stream.Seek(offset, origin); + + public override void SetLength(long value) => stream.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => stream.Write(buffer, offset, count); + + public override int ReadByte() + { + if (!hasReadAheadByte) + { + return stream.ReadByte(); + } + hasReadAheadByte = false; + return readAheadByte; + } + + public int Peek() + { + if (!hasReadAheadByte) + { + readAheadByte = stream.ReadByte(); + hasReadAheadByte = true; + } + return readAheadByte; + } + + public new void Dispose() => stream.Dispose(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/GlobalSuppressions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/GlobalSuppressions.cs new file mode 100644 index 0000000..f5096dd --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/GlobalSuppressions.cs @@ -0,0 +1,6 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:Braces should not be omitted", Justification = "This is a valid code style")] diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/CannedHttpClientFactory.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/CannedHttpClientFactory.cs new file mode 100644 index 0000000..530bb18 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/CannedHttpClientFactory.cs @@ -0,0 +1,18 @@ +using System; +using System.Net.Http; + +namespace YPermitin.SQLCLR.ClickHouseClient.Http +{ + internal class CannedHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient client; + + public CannedHttpClientFactory(HttpClient client) + { + this.client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public HttpClient CreateClient(string name) => client; + } + +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/DefaultPoolHttpClientFactory.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/DefaultPoolHttpClientFactory.cs new file mode 100644 index 0000000..b33dc00 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/DefaultPoolHttpClientFactory.cs @@ -0,0 +1,15 @@ +using System; +using System.Net; +using System.Net.Http; + +namespace YPermitin.SQLCLR.ClickHouseClient.Http +{ + internal class DefaultPoolHttpClientFactory : IHttpClientFactory + { + private static readonly HttpClientHandler DefaultHttpClientHandler = new() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; + + public TimeSpan Timeout { get; init; } + + public HttpClient CreateClient(string name) => new(DefaultHttpClientHandler, false) { Timeout = Timeout }; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/IHttpClientFactory.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/IHttpClientFactory.cs new file mode 100644 index 0000000..c3f673e --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/IHttpClientFactory.cs @@ -0,0 +1,9 @@ +using System.Net.Http; + +namespace YPermitin.SQLCLR.ClickHouseClient.Http +{ + public interface IHttpClientFactory + { + HttpClient CreateClient(string name); + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/SingleConnectionHttpClientFactory.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/SingleConnectionHttpClientFactory.cs new file mode 100644 index 0000000..53fef3c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Http/SingleConnectionHttpClientFactory.cs @@ -0,0 +1,29 @@ +using System; +using System.Net; +using System.Net.Http; + +namespace YPermitin.SQLCLR.ClickHouseClient.Http +{ + internal class SingleConnectionHttpClientFactory : IHttpClientFactory, IDisposable + { + private readonly HttpClientHandler handler; + + public TimeSpan Timeout { get; init; } + + public SingleConnectionHttpClientFactory() + { + handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + MaxConnectionsPerServer = 1, + }; + } + + public HttpClient CreateClient(string name) => new(handler, false) + { + Timeout = Timeout, + }; + + public void Dispose() => handler.Dispose(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseCommand.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseCommand.cs new file mode 100644 index 0000000..ef52b0c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseCommand.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Readers; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + public interface IClickHouseCommand : IDbCommand + { + new ClickHouseDbParameter CreateParameter(); + + Task ExecuteRawResultAsync(CancellationToken cancellationToken); + + IDictionary CustomSettings { get; } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseConnection.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseConnection.cs new file mode 100644 index 0000000..a40ee7d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseConnection.cs @@ -0,0 +1,10 @@ +using System.Data; +using YPermitin.SQLCLR.ClickHouseClient.ADO; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + public interface IClickHouseConnection : IDbConnection + { + new ClickHouseCommand CreateCommand(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseDataSource.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseDataSource.cs new file mode 100644 index 0000000..11a7923 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/IClickHouseDataSource.cs @@ -0,0 +1,17 @@ +#if NET7_0_OR_GREATER +using System.Threading; +using System.Threading.Tasks; + +namespace ClickHouse.Client; + +public interface IClickHouseDataSource +{ + string ConnectionString { get; } + + IClickHouseConnection CreateConnection(); + + IClickHouseConnection OpenConnection(); + + Task OpenConnectionAsync(CancellationToken cancellationToken = default); +} +#endif diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Json/SnakeCaseNamingPolicy.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Json/SnakeCaseNamingPolicy.cs new file mode 100644 index 0000000..fcab44f --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Json/SnakeCaseNamingPolicy.cs @@ -0,0 +1,17 @@ +/* +using System.Text.Json; +using ClickHouse.Client.Utility; + +namespace ClickHouse.Client.Json +{ + internal class SnakeCaseNamingPolicy : JsonNamingPolicy + { + public static SnakeCaseNamingPolicy Instance { get; } = new SnakeCaseNamingPolicy(); + + public override string ConvertName(string name) + { + return name.ToSnakeCase(); + } + } +} +*/ \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Numerics/ClickHouseDecimal.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Numerics/ClickHouseDecimal.cs new file mode 100644 index 0000000..c69f369 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Numerics/ClickHouseDecimal.cs @@ -0,0 +1,438 @@ +using System; +using System.Globalization; +using System.Numerics; +using System.Text; + +namespace YPermitin.SQLCLR.ClickHouseClient.Numerics +{ + /// + /// Arbitrary precision decimal. + /// All operations are exact, except for division. Division never determines more digits than the given precision. + /// Based on: https://gist.github.com/JcBernack/0b4eef59ca97ee931a2f45542b9ff06d + /// Based on https://stackoverflow.com/a/4524254 + /// Original Author: Jan Christoph Bernack (contact: jc.bernack at gmail.com) + /// + public readonly struct ClickHouseDecimal + : IComparable, IComparable, IFormattable, IConvertible, IEquatable, IComparable + { + /// + /// Sets the global maximum precision of division operations. + /// + public static int MaxDivisionPrecision = 50; + + public ClickHouseDecimal(decimal value) + : this() + { + // Slightly wasteful, but seems to be the cheapest way to get scale + var parts = decimal.GetBits(value); + int scale = (parts[3] >> 16) & 0x7F; + bool negative = (parts[3] & 0x80000000) != 0; + + var data = new byte[(3 * sizeof(int)) + 1]; + WriteIntToArray(parts[0], data, 0); + WriteIntToArray(parts[1], data, sizeof(int)); + WriteIntToArray(parts[2], data, 2 * sizeof(int)); + + var mantissa = new BigInteger(data); + if (negative) + mantissa = BigInteger.Negate(mantissa); + + Mantissa = mantissa; + Scale = scale; + } + + public ClickHouseDecimal(BigInteger mantissa, int scale) + : this() + { + if (scale < 0) + throw new ArgumentException("Scale cannot be <0", nameof(scale)); + // Normalize(ref mantissa, ref scale); + + Mantissa = mantissa; + Scale = scale; + } + + public readonly BigInteger Mantissa { get; } + + public readonly int Scale { get; } + + public static ClickHouseDecimal Zero => new(0, 0); + + public static ClickHouseDecimal One => new(1, 0); + + public int Sign => Mantissa.Sign; + + /// + /// Removes trailing zeros on the mantissa + /// + private static void Normalize(ref BigInteger mantissa, ref int scale) + { + if (mantissa.IsZero) + { + scale = 0; + } + else + { + BigInteger remainder = 0; + while (remainder == 0 && scale > 0) + { + var shortened = BigInteger.DivRem(mantissa, 10, out remainder); + if (remainder == 0) + { + mantissa = shortened; + scale--; + } + } + } + } + + /// + /// Truncate the number to the given precision by removing the least significant digits. + /// + private static void Truncate(ref BigInteger mantissa, ref int scale, int precision) + { + // remove the least significant digits, as long as the number of digits is higher than the given Precision + int digits = NumberOfDigits(mantissa); + int digitsToRemove = Math.Max(digits - precision, 0); + digitsToRemove = Math.Min(digitsToRemove, scale); + mantissa /= BigInteger.Pow(10, digitsToRemove); + scale -= digitsToRemove; + } + + public ClickHouseDecimal Truncate(int precision = 0) + { + var mantissa = Mantissa; + var scale = Scale; + Truncate(ref mantissa, ref scale, precision); + return new ClickHouseDecimal(mantissa, scale); + } + + public ClickHouseDecimal Floor() + { + return Truncate(NumberOfDigits(Mantissa) - Scale); + } + + public static int NumberOfDigits(BigInteger value) => value == 0 ? 0 : (int)Math.Ceiling(BigInteger.Log10(value * value.Sign)); + + public static implicit operator ClickHouseDecimal(int value) => new ClickHouseDecimal(value, 0); + + public static implicit operator ClickHouseDecimal(double value) + { + var mantissa = (BigInteger)value; + int scale = 0; + double scaleFactor = 1; + while (Math.Abs((value * scaleFactor) - (double)mantissa) > 0) + { + scale += 1; + scaleFactor *= 10; + mantissa = (BigInteger)(value * scaleFactor); + } + return new ClickHouseDecimal(mantissa, scale); + } + + public static implicit operator ClickHouseDecimal(decimal value) + { + return new ClickHouseDecimal(value); + } + + public static explicit operator double(ClickHouseDecimal value) + { + return (double)value.Mantissa / Math.Pow(10, value.Scale); + } + + public static explicit operator float(ClickHouseDecimal value) + { + return Convert.ToSingle((double)value); + } + + public static explicit operator decimal(ClickHouseDecimal value) + { + var mantissa = value.Mantissa; + var scale = value.Scale; + + bool negative = mantissa < 0; + if (negative) + { + mantissa = BigInteger.Negate(mantissa); + } + + var numberBytes = mantissa.ToByteArray(); + switch (numberBytes.Length) + { + case 13 when numberBytes[12] == 0: + break; + case (> 12): + ThrowDecimalOverflowException(); + break; + default: + break; + } + + var data = new byte[3 * sizeof(int)]; + Buffer.BlockCopy(numberBytes, 0, data, 0, Math.Min(numberBytes.Length, 12)); + + int part0 = BitConverter.ToInt32(data, 0); + int part1 = BitConverter.ToInt32(data, 4); + int part2 = BitConverter.ToInt32(data, 8); + + var result = new decimal(part0, part1, part2, negative, (byte)scale); + return result; + } + + public static explicit operator int(ClickHouseDecimal value) + { + return (int)(value.Mantissa / BigInteger.Pow(10, value.Scale)); + } + + public static explicit operator uint(ClickHouseDecimal value) + { + return (uint)(value.Mantissa / BigInteger.Pow(10, value.Scale)); + } + + public static explicit operator long(ClickHouseDecimal value) + { + return (long)(value.Mantissa / BigInteger.Pow(10, value.Scale)); + } + + public static explicit operator ulong(ClickHouseDecimal value) + { + return (ulong)(value.Mantissa / BigInteger.Pow(10, value.Scale)); + } + + public static ClickHouseDecimal operator +(ClickHouseDecimal left, ClickHouseDecimal right) + { + var scale = Math.Max(left.Scale, right.Scale); + var left_mantissa = ScaleMantissa(left, scale); + var right_mantissa = ScaleMantissa(right, scale); + + return new ClickHouseDecimal(left_mantissa + right_mantissa, scale); + } + + public static ClickHouseDecimal operator -(ClickHouseDecimal left, ClickHouseDecimal right) + { + var scale = Math.Max(left.Scale, right.Scale); + var left_mantissa = ScaleMantissa(left, scale); + var right_mantissa = ScaleMantissa(right, scale); + + return new ClickHouseDecimal(left_mantissa - right_mantissa, scale); + } + + public static ClickHouseDecimal operator *(ClickHouseDecimal left, ClickHouseDecimal right) + { + return new ClickHouseDecimal(left.Mantissa * right.Mantissa, left.Scale + right.Scale); + } + + public static ClickHouseDecimal operator /(ClickHouseDecimal dividend, ClickHouseDecimal divisor) + { + var dividend_mantissa = dividend.Mantissa; + var divisor_mantissa = divisor.Mantissa; + + var bias = MaxDivisionPrecision - (NumberOfDigits(dividend_mantissa) - NumberOfDigits(divisor_mantissa)); + bias = Math.Max(0, bias); + + dividend_mantissa *= BigInteger.Pow(10, bias); + + var result_mantissa = dividend_mantissa / divisor_mantissa; + var result_scale = dividend.Scale - divisor.Scale + bias; + Normalize(ref result_mantissa, ref result_scale); + return new ClickHouseDecimal(result_mantissa, result_scale); + } + + public static ClickHouseDecimal operator %(ClickHouseDecimal dividend, ClickHouseDecimal divisor) + { + var scale = Math.Max(dividend.Scale, divisor.Scale); + var dividend_mantissa = ScaleMantissa(dividend, scale); + var divisor_mantissa = ScaleMantissa(divisor, scale); + + return new ClickHouseDecimal(dividend_mantissa % divisor_mantissa, scale); + } + + public static bool operator ==(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) == 0; + } + + public static bool operator !=(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) != 0; + } + + public static bool operator <(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) < 0; + } + + public static bool operator >(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) > 0; + } + + public static bool operator <=(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) <= 0; + } + + public static bool operator >=(ClickHouseDecimal left, ClickHouseDecimal right) + { + return left.CompareTo(right) >= 0; + } + + public bool Equals(ClickHouseDecimal other) + { + var maxScale = Math.Max(Scale, other.Scale); + + return ScaleMantissa(this, maxScale) == ScaleMantissa(other, maxScale); + } + + public override bool Equals(object obj) => CompareTo(obj) == 0; + + public override int GetHashCode() + { + unchecked + { + return (Mantissa.GetHashCode() * 397) ^ Scale; + } + } + + public int CompareTo(object obj) + { + return obj is ClickHouseDecimal cbi ? CompareTo(cbi) : 1; + } + + public int CompareTo(ClickHouseDecimal other) + { + var maxScale = Math.Max(Scale, other.Scale); + var left_mantissa = ScaleMantissa(this, maxScale); + var right_mantissa = ScaleMantissa(other, maxScale); + + return left_mantissa.CompareTo(right_mantissa); + } + + public string ToString(string format, IFormatProvider formatProvider) => ToString(formatProvider); + + public string ToString(IFormatProvider provider) + { + provider ??= CultureInfo.CurrentCulture; + var numberFormat = (NumberFormatInfo)provider.GetFormat(typeof(NumberFormatInfo)); + var builder = new StringBuilder(); + + var mantissa = Mantissa; + if (mantissa < 0) + { + builder.Append(numberFormat.NegativeSign); + mantissa = BigInteger.Negate(mantissa); + } + + if (Scale > 0) + { + var factor = BigInteger.Pow(10, Scale); + var wholePart = mantissa / factor; + var fractionalPart = mantissa - (wholePart * factor); + builder.Append(wholePart.ToString(provider)); + builder.Append(numberFormat.NumberDecimalSeparator); + builder.Append(fractionalPart.ToString(provider).PadLeft(Scale, '0')); + } + else + { + builder.Append(mantissa.ToString(provider)); + } + return builder.ToString(); + } + + public static ClickHouseDecimal Parse(string input) => Parse(input, CultureInfo.CurrentCulture); + + public static ClickHouseDecimal Parse(string input, IFormatProvider provider) + { + var numberFormat = (NumberFormatInfo)provider.GetFormat(typeof(NumberFormatInfo)); + + if (string.IsNullOrWhiteSpace(input)) + { + return Zero; + } + input = input.Trim(); + + string mantissaPart = input; + string fractionalPart = string.Empty; + + int separatorIndex = input.IndexOf(numberFormat.NumberDecimalSeparator, StringComparison.InvariantCultureIgnoreCase); + if (separatorIndex > 0) + { + fractionalPart = input.Substring(separatorIndex + 1); + mantissaPart = input.Replace(numberFormat.NumberDecimalSeparator, string.Empty); + } + var mantissa = BigInteger.Parse(mantissaPart, NumberStyles.Any, provider); + var scale = fractionalPart.Length; + + return new ClickHouseDecimal(mantissa, scale); + } + + public override string ToString() => ToString(null, CultureInfo.CurrentCulture); + + public TypeCode GetTypeCode() => TypeCode.Object; + + public bool ToBoolean(IFormatProvider provider) => !Mantissa.IsZero; + + public char ToChar(IFormatProvider provider) => (char)(int)this; + + public sbyte ToSByte(IFormatProvider provider) => (sbyte)(int)this; + + public byte ToByte(IFormatProvider provider) => (byte)(int)this; + + public short ToInt16(IFormatProvider provider) => (short)(int)this; + + public ushort ToUInt16(IFormatProvider provider) => (ushort)(uint)this; + + public int ToInt32(IFormatProvider provider) => (short)(int)this; + + public uint ToUInt32(IFormatProvider provider) => (uint)this; + + public long ToInt64(IFormatProvider provider) => (long)this; + + public ulong ToUInt64(IFormatProvider provider) => (ulong)this; + + public float ToSingle(IFormatProvider provider) => (float)this; + + public double ToDouble(IFormatProvider provider) => (double)this; + + public decimal ToDecimal(IFormatProvider provider) => (decimal)this; + + public DateTime ToDateTime(IFormatProvider provider) => throw new NotSupportedException(); + + public object ToType(Type conversionType, IFormatProvider provider) + { + if (conversionType == typeof(BigInteger)) + { + var mantissa = this.Mantissa; + var scale = this.Scale; + Truncate(ref mantissa, ref scale, 0); + return mantissa; + } + return Convert.ChangeType(this, conversionType); + } + + public int CompareTo(decimal other) => CompareTo((ClickHouseDecimal)other); + + internal static BigInteger ScaleMantissa(ClickHouseDecimal value, int scale) + { + if (scale == value.Scale) + return value.Mantissa; + if (scale < value.Scale) + return value.Mantissa / BigInteger.Pow(10, value.Scale - scale); + return value.Mantissa * BigInteger.Pow(10, scale - value.Scale); + } + + private static void WriteIntToArray(int value, byte[] array, int index) + { + array[index + 0] = (byte)value; + array[index + 1] = (byte)(value >> 8); + array[index + 2] = (byte)(value >> 0x10); + array[index + 3] = (byte)(value >> 0x18); + } + + // [DoesNotReturn] + private static void ThrowDecimalOverflowException() + { + throw new OverflowException("Value cannot be represented as System.Decimal"); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Properties/AssemblyInfo.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..55a101f --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Properties/AssemblyInfo.cs @@ -0,0 +1,33 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// Общие сведения об этой сборке предоставляются следующим набором +// набора атрибутов. Измените значения этих атрибутов для изменения сведений, +// связанные со сборкой. +[assembly: AssemblyTitle("ClickHouseClientLegacy")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ClickHouseClientLegacy")] +[assembly: AssemblyCopyright("Copyright © 2024")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Установка значения False для параметра ComVisible делает типы в этой сборке невидимыми +// для компонентов COM. Если необходимо обратиться к типу в этой сборке через +// COM, задайте атрибуту ComVisible значение TRUE для этого типа. +[assembly: ComVisible(false)] + +// Следующий GUID служит для идентификации библиотеки типов, если этот проект будет видимым для COM +[assembly: Guid("de8e7d56-8a86-4142-84a7-5ceb3162329f")] + +// Сведения о версии сборки состоят из указанных ниже четырех значений: +// +// Основной номер версии +// Дополнительный номер версии +// Номер сборки +// Редакция +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/TypeSettings.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/TypeSettings.cs new file mode 100644 index 0000000..316036d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/TypeSettings.cs @@ -0,0 +1,12 @@ +using NodaTime; + +namespace YPermitin.SQLCLR.ClickHouseClient +{ + // удалено record struct + internal record TypeSettings(bool useBigDecimal, string timezone) + { + public static string DefaultTimezone = DateTimeZoneProviders.Tzdb.GetSystemDefault().Id; + + public static TypeSettings Default => new TypeSettings(true, DefaultTimezone); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractBigIntegerType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractBigIntegerType.cs new file mode 100644 index 0000000..01bfbbb --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractBigIntegerType.cs @@ -0,0 +1,68 @@ +using System; +using System.Globalization; +using System.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal abstract class AbstractBigIntegerType : IntegerType + { + public virtual int Size { get; } + + public override Type FrameworkType => typeof(BigInteger); + + public override object Read(ExtendedBinaryReader reader) + { + if (Signed) + return new BigInteger(reader.ReadBytes(Size)); + + var data = new byte[Size + 1]; + for (int i = 0; i < Size; i++) + data[i] = reader.ReadByte(); + data[Size] = 0; + return new BigInteger(data); + } + + public abstract override string ToString(); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var bigInt = value switch + { + BigInteger bi => bi, + decimal dl => new BigInteger(dl), + double d => new BigInteger(d), + float f => new BigInteger(f), + int i => new BigInteger(i), + uint ui => new BigInteger(ui), + long l => new BigInteger(l), + ulong ul => new BigInteger(ul), + _ => new BigInteger(Convert.ToInt64(value, CultureInfo.InvariantCulture)) + }; + + if (bigInt < 0 && !Signed) + throw new ArgumentException("Cannot convert negative BigInteger to UInt"); + + byte[] bigIntBytes = bigInt.ToByteArray(); + byte[] decimalBytes = new byte[Size]; + + var lengthToCopy = bigIntBytes.Length; + if (!Signed && bigIntBytes[bigIntBytes.Length - 1] == 0) + lengthToCopy = bigIntBytes.Length - 1; + + if (lengthToCopy > Size) + throw new OverflowException($"Got {lengthToCopy} bytes, {Size} expected"); + + Array.Copy(bigIntBytes, decimalBytes, lengthToCopy); + + // If a negative BigInteger is not long enough to fill the whole buffer, + // the remainder needs to be filled with 0xFF + if (bigInt < 0) + { + for (int i = bigIntBytes.Length; i < Size; i++) + decimalBytes[i] = 0xFF; + } + writer.Write(decimalBytes); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractDateTimeType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractDateTimeType.cs new file mode 100644 index 0000000..d623856 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AbstractDateTimeType.cs @@ -0,0 +1,59 @@ +using System; +using NodaTime; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + public static class DateTimeConversions + { + public static readonly DateTime DateTimeEpochStart = DateTimeOffset.FromUnixTimeSeconds(0).UtcDateTime; + +#if NET6_0_OR_GREATER + public static readonly DateOnly DateOnlyEpochStart = new(1970, 1, 1); +#endif + + public static int ToUnixTimeDays(this DateTimeOffset dto) + { + return (int)(dto.Date - DateTimeEpochStart.Date).TotalDays; + } + + public static DateTime FromUnixTimeDays(int days) => DateTimeEpochStart.AddDays(days); + } + + internal abstract class AbstractDateTimeType : ParameterizedType + { + public DateTimeOffset CoerceToDateTimeOffset(object value) + { + return value switch + { +#if NET6_0_OR_GREATER + DateOnly date => new DateTimeOffset(date.Year, date.Month, date.Day, 0, 0, 0, TimeSpan.Zero), +#endif + DateTimeOffset v => v, + DateTime dt => TimeZoneOrUtc.AtLeniently(LocalDateTime.FromDateTime(dt)).ToDateTimeOffset(), + OffsetDateTime o => o.ToDateTimeOffset(), + ZonedDateTime z => z.ToDateTimeOffset(), + Instant i => ToDateTimeOffset(i), + _ => throw new NotSupportedException() + }; + } + + public override Type FrameworkType => typeof(DateTime); + + public DateTimeZone TimeZone { get; set; } + + public DateTimeZone TimeZoneOrUtc => TimeZone ?? DateTimeZone.Utc; + + public override string ToString() => TimeZone == null ? $"{Name}" : $"{Name}({TimeZone.Id})"; + + private DateTimeOffset ToDateTimeOffset(Instant instant) => instant.InZone(TimeZoneOrUtc).ToDateTimeOffset(); + + public DateTime ToDateTime(Instant instant) + { + var zonedDateTime = instant.InZone(TimeZoneOrUtc); + if (zonedDateTime.Offset.Ticks == 0) + return zonedDateTime.ToDateTimeUtc(); + else + return zonedDateTime.ToDateTimeUnspecified(); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AggregateFunctionType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AggregateFunctionType.cs new file mode 100644 index 0000000..2168981 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/AggregateFunctionType.cs @@ -0,0 +1,38 @@ +using System; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class AggregateFunctionType : ParameterizedType + { + public string Function { get; private set; } + + public override string Name => "AggregateFunction"; + + + + public override Type FrameworkType => throw new AggregateFunctionException(Function); + + public override ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new AggregateFunctionType { Function = typeName.ChildNodes.First().Value }; + } + + public override object Read(ExtendedBinaryReader reader) => throw new AggregateFunctionException(Function); + + public override string ToString() => throw new AggregateFunctionException(Function); + + public override void Write(ExtendedBinaryWriter writer, object value) => throw new AggregateFunctionException(Function); + + [Serializable] + public class AggregateFunctionException : Exception + { + public AggregateFunctionException(string function) + : base($"Unable to directly query column with type AggregateFunction({function}). Use {function}Merge() function to query this value") + { + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ArrayType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ArrayType.cs new file mode 100644 index 0000000..6becc66 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ArrayType.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class ArrayType : ParameterizedType + { + public ClickHouseType UnderlyingType { get; set; } + + public override Type FrameworkType => UnderlyingType.FrameworkType.MakeArrayType(); + + public override string Name => "Array"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new ArrayType + { + UnderlyingType = parseClickHouseTypeFunc(node.SingleChild), + }; + } + + public override string ToString() => $"{Name}({UnderlyingType})"; + + public override object Read(ExtendedBinaryReader reader) + { + var length = reader.Read7BitEncodedInt(); + var data = Array.CreateInstance(UnderlyingType.FrameworkType, length); + for (var i = 0; i < length; i++) + { + data.SetValue(ClearDBNull(UnderlyingType.Read(reader)), i); + } + return data; + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { + if (value is null || value is DBNull) + { + writer.Write7BitEncodedInt(0); + return; + } + + var collection = (IList)value; + writer.Write7BitEncodedInt(collection.Count); + for (var i = 0; i < collection.Count; i++) + { + UnderlyingType.Write(writer, collection[i]); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/BooleanType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/BooleanType.cs new file mode 100644 index 0000000..fb8d6e8 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/BooleanType.cs @@ -0,0 +1,16 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class BooleanType : ClickHouseType + { + public override Type FrameworkType => typeof(bool); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadBoolean(); + + public override string ToString() => "Bool"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write((bool)value); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ClickHouseType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ClickHouseType.cs new file mode 100644 index 0000000..adbf6ef --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ClickHouseType.cs @@ -0,0 +1,18 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal abstract class ClickHouseType + { + public abstract Type FrameworkType { get; } + + public abstract object Read(ExtendedBinaryReader reader); + + public abstract void Write(ExtendedBinaryWriter writer, object value); + + public abstract override string ToString(); + + protected static object ClearDBNull(object value) => value is DBNull ? null : value; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Date32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Date32Type.cs new file mode 100644 index 0000000..e755de8 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Date32Type.cs @@ -0,0 +1,22 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Date32Type : DateType + { + public override string Name { get; } + + public override string ToString() => "Date32"; + + public override object Read(ExtendedBinaryReader reader) => DateTimeConversions.FromUnixTimeDays(reader.ReadInt32()); + + public override ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings) => throw new NotImplementedException(); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + writer.Write(CoerceToDateTimeOffset(value).ToUnixTimeDays()); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime32Type.cs new file mode 100644 index 0000000..0fbc366 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime32Type.cs @@ -0,0 +1,7 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class DateTime32Type : DateTimeType + { + public override string Name => "DateTime32"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime64Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime64Type.cs new file mode 100644 index 0000000..549125d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTime64Type.cs @@ -0,0 +1,53 @@ +using System; +using System.Globalization; +using NodaTime; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class DateTime64Type : AbstractDateTimeType + { + public int Scale { get; set; } + + public override string Name => "DateTime64"; + + public override string ToString() => TimeZone == null ? $"DateTime64({Scale})" : $"DateTime64({Scale}, {TimeZone.Id})"; + + public DateTime FromClickHouseTicks(long clickHouseTicks) + { + // Convert ClickHouse variable precision ticks into "standard" .NET 100ns ones + var ticks = MathUtils.ShiftDecimalPlaces(clickHouseTicks, 7 - Scale); + return ToDateTime(Instant.FromUnixTimeTicks(ticks)); + } + + public long ToClickHouseTicks(Instant instant) => MathUtils.ShiftDecimalPlaces(instant.ToUnixTimeTicks(), Scale - 7); + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + var scale = int.Parse(node.ChildNodes[0].Value, CultureInfo.InvariantCulture); + + DateTimeZone timeZone = null; + if (node.ChildNodes.Count > 1) + { + var timeZoneName = node.ChildNodes[1].Value.Trim('\''); + timeZone = DateTimeZoneProviders.Tzdb.GetZoneOrNull(timeZoneName); + } + timeZone ??= DateTimeZoneProviders.Tzdb.GetZoneOrNull(settings.timezone); + + return new DateTime64Type + { + TimeZone = timeZone, + Scale = scale, + }; + } + + public override object Read(ExtendedBinaryReader reader) => FromClickHouseTicks(reader.ReadInt64()); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + writer.Write(ToClickHouseTicks(Instant.FromDateTimeOffset(CoerceToDateTimeOffset(value)))); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTimeType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTimeType.cs new file mode 100644 index 0000000..a4bf13c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateTimeType.cs @@ -0,0 +1,32 @@ +using System; +using NodaTime; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class DateTimeType : AbstractDateTimeType + { + public override string Name => "DateTime"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + DateTimeZone timeZone = null; + if (node.ChildNodes.Count > 0) + { + var timeZoneName = node.ChildNodes[0].Value.Trim('\''); + timeZone = DateTimeZoneProviders.Tzdb.GetZoneOrNull(timeZoneName); + } + timeZone ??= DateTimeZoneProviders.Tzdb.GetZoneOrNull(settings.timezone); + + return new DateTimeType { TimeZone = timeZone }; + } + + public override object Read(ExtendedBinaryReader reader) => ToDateTime(Instant.FromUnixTimeSeconds(reader.ReadUInt32())); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + writer.Write((int)CoerceToDateTimeOffset(value).ToUnixTimeSeconds()); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateType.cs new file mode 100644 index 0000000..bc2f954 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DateType.cs @@ -0,0 +1,22 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class DateType : AbstractDateTimeType + { + public override string Name { get; } + + public override string ToString() => "Date"; + + public override object Read(ExtendedBinaryReader reader) => DateTimeConversions.FromUnixTimeDays(reader.ReadUInt16()); + + public override ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings) => throw new NotImplementedException(); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + writer.Write(Convert.ToUInt16(CoerceToDateTimeOffset(value).ToUnixTimeDays())); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal128Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal128Type.cs new file mode 100644 index 0000000..f1e9b8d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal128Type.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Decimal128Type : DecimalType + { + public Decimal128Type() + { + Precision = 38; + } + + public override int Size => 16; + + public override string Name => "Decimal128"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new Decimal128Type + { + Scale = int.Parse(node.SingleChild.Value, CultureInfo.InvariantCulture), + UseBigDecimal = settings.useBigDecimal, + }; + } + + public override string ToString() => $"{Name}({Scale})"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal256Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal256Type.cs new file mode 100644 index 0000000..f91e33b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal256Type.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Decimal256Type : DecimalType + { + public Decimal256Type() + { + Precision = 76; + } + + public override int Size => 32; + + public override string Name => "Decimal256"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new Decimal256Type + { + Scale = int.Parse(node.SingleChild.Value, CultureInfo.InvariantCulture), + UseBigDecimal = settings.useBigDecimal, + }; + } + + public override string ToString() => $"{Name}({Scale})"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal32Type.cs new file mode 100644 index 0000000..fb20e6b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal32Type.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Decimal32Type : DecimalType + { + public Decimal32Type() + { + Precision = 9; + } + + public override string Name => "Decimal32"; + + public override int Size => 4; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new Decimal32Type + { + Scale = int.Parse(node.SingleChild.Value, CultureInfo.InvariantCulture), + UseBigDecimal = settings.useBigDecimal, + }; + } + + public override string ToString() => $"{Name}({Scale})"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal64Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal64Type.cs new file mode 100644 index 0000000..c994474 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Decimal64Type.cs @@ -0,0 +1,26 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Decimal64Type : DecimalType + { + public Decimal64Type() + { + Precision = 18; + } + + public override int Size => 8; + + public override string Name => "Decimal64"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) => new Decimal64Type + { + Scale = int.Parse(node.SingleChild.Value, CultureInfo.InvariantCulture), + UseBigDecimal = settings.useBigDecimal, + }; + + public override string ToString() => $"{Name}({Scale})"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DecimalType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DecimalType.cs new file mode 100644 index 0000000..c05b3e5 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/DecimalType.cs @@ -0,0 +1,135 @@ +using System; +using System.Globalization; +using System.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class DecimalType : ParameterizedType + { + private int scale; + + public virtual int Precision { get; init; } + + /// + /// Gets or sets the decimal 'scale' (precision) in ClickHouse + /// + public int Scale + { + get => scale; + set + { + scale = value; + Exponent = BigInteger.Pow(10, value); + } + } + + /// + /// Gets decimal exponent value based on Scale + /// + public BigInteger Exponent { get; private set; } + + public override string Name => "Decimal"; + + /// + /// Gets size of type in bytes + /// + public virtual int Size => GetSizeFromPrecision(Precision); + + public override Type FrameworkType => UseBigDecimal ? typeof(ClickHouseDecimal) : typeof(decimal); + + public ClickHouseDecimal MaxValue => new(BigInteger.Pow(10, Precision) - 1, Scale); + + public ClickHouseDecimal MinValue => new(1 - BigInteger.Pow(10, Precision), Scale); + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + var precision = int.Parse(node.ChildNodes[0].Value, CultureInfo.InvariantCulture); + var scale = int.Parse(node.ChildNodes[1].Value, CultureInfo.InvariantCulture); + + var size = GetSizeFromPrecision(precision); + + return size switch + { + 4 => new Decimal32Type { Precision = precision, Scale = scale, UseBigDecimal = settings.useBigDecimal }, + 8 => new Decimal64Type { Precision = precision, Scale = scale, UseBigDecimal = settings.useBigDecimal }, + 16 => new Decimal128Type { Precision = precision, Scale = scale, UseBigDecimal = settings.useBigDecimal }, + 32 => new Decimal256Type { Precision = precision, Scale = scale, UseBigDecimal = settings.useBigDecimal }, + _ => new DecimalType { Precision = precision, Scale = scale, UseBigDecimal = settings.useBigDecimal }, + }; + } + + public override object Read(ExtendedBinaryReader reader) + { + if (UseBigDecimal) + { + var mantissa = Size switch + { + 4 => (BigInteger)reader.ReadInt32(), + 8 => (BigInteger)reader.ReadInt64(), + _ => new BigInteger(reader.ReadBytes(Size)), + }; + return new ClickHouseDecimal(mantissa, Scale); + } + else + { + var mantissa = Size switch + { + 4 => reader.ReadInt32(), + 8 => reader.ReadInt64(), + _ => (decimal)new BigInteger(reader.ReadBytes(Size)), + }; + return mantissa / (decimal)Exponent; + } + } + + public override string ToString() => $"{Name}({Precision}, {Scale})"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + try + { + ClickHouseDecimal @decimal = value is ClickHouseDecimal chd ? chd : Convert.ToDecimal(value, CultureInfo.InvariantCulture); + var mantissa = ClickHouseDecimal.ScaleMantissa(@decimal, Scale); + WriteBigInteger(writer, mantissa); + } + catch (OverflowException) + { + throw new ArgumentOutOfRangeException(nameof(value), value, $"Value cannot be represented"); + } + } + + protected virtual bool UseBigDecimal { get; init; } + + private static int GetSizeFromPrecision(int precision) => precision switch + { + int p when p >= 1 && p <= 9 => 4, + int p when p >= 10 && p <= 18 => 8, + int p when p >= 19 && p <= 38 => 16, + int p when p >= 39 && p <= 76 => 32, + _ => throw new ArgumentOutOfRangeException(nameof(precision)), + }; + + private void WriteBigInteger(ExtendedBinaryWriter writer, BigInteger value) + { + byte[] bigIntBytes = value.ToByteArray(); + byte[] decimalBytes = new byte[Size]; + + if (bigIntBytes.Length > Size) + throw new OverflowException($"Trying to write {bigIntBytes.Length} bytes, at most {Size} expected"); + + bigIntBytes.CopyTo(decimalBytes, 0); + + // If a negative BigInteger is not long enough to fill the whole buffer, + // the remainder needs to be filled with 0xFF + if (value.Sign < 0) + { + for (int i = bigIntBytes.Length; i < Size; i++) + decimalBytes[i] = 0xFF; + } + writer.Write(decimalBytes); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum16Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum16Type.cs new file mode 100644 index 0000000..1a2297b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum16Type.cs @@ -0,0 +1,21 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Enum16Type : EnumType + { + public override string Name => "Enum16"; + + public override string ToString() => "Enum16"; + + public override object Read(ExtendedBinaryReader reader) => Lookup(reader.ReadInt16()); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var enumIndex = value is string enumStr ? (short)Lookup(enumStr) : Convert.ToInt16(value, CultureInfo.InvariantCulture); + writer.Write(enumIndex); + } + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum8Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum8Type.cs new file mode 100644 index 0000000..05a409c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Enum8Type.cs @@ -0,0 +1,19 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Enum8Type : EnumType + { + public override string Name => "Enum8"; + + public override object Read(ExtendedBinaryReader reader) => Lookup(reader.ReadSByte()); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var enumIndex = value is string enumStr ? (sbyte)Lookup(enumStr) : Convert.ToSByte(value, CultureInfo.InvariantCulture); + writer.Write(enumIndex); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/EnumType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/EnumType.cs new file mode 100644 index 0000000..4bf48da --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/EnumType.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class EnumType : ParameterizedType + { + private Dictionary values = new Dictionary(); + + public override string Name => "Enum"; + + public override Type FrameworkType => typeof(string); + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + var parameters = node.ChildNodes + .Select(cn => cn.Value) + .Select(p => p.Split('=')) + .ToDictionary(kvp => kvp[0].Trim().Trim('\''), kvp => Convert.ToInt32(kvp[1].Trim(), CultureInfo.InvariantCulture)); + + switch (node.Value) + { + case "Enum": + case "Enum8": + return new Enum8Type { values = parameters }; + case "Enum16": + return new Enum16Type { values = parameters }; + default: throw new ArgumentOutOfRangeException($"Unsupported Enum type: {node.Value}"); + } + } + + public int Lookup(string key) => values[key]; + + public string Lookup(int value) => values.SingleOrDefault(kvp => kvp.Value == value).Key ?? throw new KeyNotFoundException(); + + public override string ToString() => $"{Name}({string.Join(",", values.Select(kvp => kvp.Key + "=" + kvp.Value))}"; + + public override object Read(ExtendedBinaryReader reader) => throw new NotImplementedException(); + + public override void Write(ExtendedBinaryWriter writer, object value) => throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FixedStringType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FixedStringType.cs new file mode 100644 index 0000000..429cf7b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FixedStringType.cs @@ -0,0 +1,37 @@ +using System; +using System.Globalization; +using System.Text; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class FixedStringType : ParameterizedType + { + public int Length { get; set; } + + public override Type FrameworkType => typeof(string); + + public override string Name => "FixedString"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new FixedStringType + { + Length = int.Parse(node.SingleChild.Value, CultureInfo.InvariantCulture), + }; + } + + public override string ToString() => $"FixedString({Length})"; + + public override object Read(ExtendedBinaryReader reader) => Encoding.UTF8.GetString(reader.ReadBytes(Length)); + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var @string = Convert.ToString(value, CultureInfo.InvariantCulture); + var stringBytes = new byte[Length]; + Encoding.UTF8.GetBytes(@string, 0, @string.Length, stringBytes, 0); + writer.Write(stringBytes); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float32Type.cs new file mode 100644 index 0000000..b7921ee --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float32Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Float32Type : FloatType + { + public override Type FrameworkType => typeof(float); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadSingle(); + + public override string ToString() => "Float32"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToSingle(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float64Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float64Type.cs new file mode 100644 index 0000000..21ac4b8 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Float64Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Float64Type : FloatType + { + public override Type FrameworkType => typeof(double); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadDouble(); + + public override string ToString() => "Float64"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToDouble(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FloatType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FloatType.cs new file mode 100644 index 0000000..de3477a --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/FloatType.cs @@ -0,0 +1,6 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal abstract class FloatType : ClickHouseType + { + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Parser.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Parser.cs new file mode 100644 index 0000000..944f2ee --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Parser.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types.Grammar +{ + public static class Parser + { + public static SyntaxTreeNode Parse(string input) + { + var tokens = Tokenizer.GetTokens(input).ToList(); + var stack = new Stack(); + SyntaxTreeNode current = null; + + foreach (var token in tokens) + { + switch (token) + { + case "(": + stack.Push(current); + break; + case ",": + stack.Peek().ChildNodes.Add(current); + break; + case ")": + stack.Peek().ChildNodes.Add(current); + current = stack.Pop(); + break; + default: + current = new SyntaxTreeNode { Value = token }; + break; + } + } + return current; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/SyntaxTreeNode.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/SyntaxTreeNode.cs new file mode 100644 index 0000000..2f987f1 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/SyntaxTreeNode.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types.Grammar +{ + public class SyntaxTreeNode + { + public string Value { get; set; } + + public IList ChildNodes { get; } = new List(); + + public SyntaxTreeNode SingleChild => ChildNodes.Count == 1 ? ChildNodes[0] : throw new ArgumentOutOfRangeException(); + + public override string ToString() + { + var builder = new StringBuilder(); + builder.Append(Value); + if (ChildNodes.Count > 0) + { + builder.Append('('); + builder.Append(string.Join(", ", ChildNodes)); + builder.Append(')'); + } + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Tokenizer.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Tokenizer.cs new file mode 100644 index 0000000..9080618 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Grammar/Tokenizer.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types.Grammar +{ + public static class Tokenizer + { + private static readonly char[] Breaks = new[] { ',', '(', ')' }; + + public static IEnumerable GetTokens(string input) + { + var start = 0; + var len = input.Length; + + while (start < len) + { + var nextBreak = input.IndexOfAny(Breaks, start); + if (nextBreak == start) + { + start++; + yield return input.Substring(nextBreak, 1); + } + else if (nextBreak == -1) + { + yield return input.Substring(start).Trim(); + yield break; + } + else + { + yield return input.Substring(start, nextBreak - start).Trim(); + start = nextBreak; + } + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv4Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv4Type.cs new file mode 100644 index 0000000..502cbe2 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv4Type.cs @@ -0,0 +1,33 @@ +using System; +using System.Net; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class IPv4Type : ClickHouseType + { + public override Type FrameworkType => typeof(IPAddress); + + public override object Read(ExtendedBinaryReader reader) + { + var ipv4bytes = reader.ReadBytes(4); + Array.Reverse(ipv4bytes); + return new IPAddress(ipv4bytes); + } + + public override string ToString() => "IPv4"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var address4 = value is IPAddress a ? a : IPAddress.Parse((string)value); + if (address4.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork) + { + throw new ArgumentException($"Expected IPv4, got {address4.AddressFamily}"); + } + + var ipv4bytes = address4.GetAddressBytes(); + Array.Reverse(ipv4bytes); + writer.Write(ipv4bytes, 0, ipv4bytes.Length); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv6Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv6Type.cs new file mode 100644 index 0000000..2a3ae29 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IPv6Type.cs @@ -0,0 +1,27 @@ +using System; +using System.Net; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class IPv6Type : ClickHouseType + { + public override Type FrameworkType => typeof(IPAddress); + + public override object Read(ExtendedBinaryReader reader) => new IPAddress(reader.ReadBytes(16)); + + public override string ToString() => "IPv6"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var address6 = value is IPAddress a ? a : IPAddress.Parse((string)value); + if (address6.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6) + { + throw new ArgumentException($"Expected IPv6, got {address6.AddressFamily}"); + } + + var ipv6bytes = address6.GetAddressBytes(); + writer.Write(ipv6bytes, 0, ipv6bytes.Length); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int128Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int128Type.cs new file mode 100644 index 0000000..ad57f58 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int128Type.cs @@ -0,0 +1,9 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int128Type : AbstractBigIntegerType + { + public override int Size => 16; + + public override string ToString() => "Int128"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int16Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int16Type.cs new file mode 100644 index 0000000..d370f97 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int16Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int16Type : IntegerType + { + public override Type FrameworkType => typeof(short); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadInt16(); + + public override string ToString() => "Int16"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToInt16(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int256Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int256Type.cs new file mode 100644 index 0000000..119f38b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int256Type.cs @@ -0,0 +1,9 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int256Type : AbstractBigIntegerType + { + public override int Size => 32; + + public override string ToString() => "Int256"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int32Type.cs new file mode 100644 index 0000000..670bbc7 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int32Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int32Type : IntegerType + { + public override Type FrameworkType => typeof(int); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadInt32(); + + public override string ToString() => "Int32"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToInt32(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int64Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int64Type.cs new file mode 100644 index 0000000..e5a7463 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int64Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int64Type : IntegerType + { + public override Type FrameworkType => typeof(long); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadInt64(); + + public override string ToString() => "Int64"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToInt64(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int8Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int8Type.cs new file mode 100644 index 0000000..1ed027b --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/Int8Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class Int8Type : IntegerType + { + public override Type FrameworkType => typeof(sbyte); + + public override string ToString() => "Int8"; + + public override object Read(ExtendedBinaryReader reader) => reader.ReadSByte(); + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToSByte(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IntegerType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IntegerType.cs new file mode 100644 index 0000000..feebfae --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/IntegerType.cs @@ -0,0 +1,7 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal abstract class IntegerType : ClickHouseType + { + public virtual bool Signed => true; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/LowCardinalityType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/LowCardinalityType.cs new file mode 100644 index 0000000..39a9213 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/LowCardinalityType.cs @@ -0,0 +1,29 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class LowCardinalityType : ParameterizedType + { + public ClickHouseType UnderlyingType { get; set; } + + public override string Name => "LowCardinality"; + + public override Type FrameworkType => UnderlyingType.FrameworkType; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new LowCardinalityType + { + UnderlyingType = parseClickHouseTypeFunc(node.SingleChild), + }; + } + + public override string ToString() => $"{Name}({UnderlyingType})"; + + public override object Read(ExtendedBinaryReader reader) => UnderlyingType.Read(reader); + + public override void Write(ExtendedBinaryWriter writer, object value) => UnderlyingType.Write(writer, value); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MapType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MapType.cs new file mode 100644 index 0000000..0c91488 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MapType.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class MapType : ParameterizedType + { + private Type frameworkType; + private ClickHouseType keyType; + private ClickHouseType valueType; + + public Tuple UnderlyingTypes + { + get => Tuple.Create(keyType, valueType); + + set + { + keyType = value.Item1; + valueType = value.Item2; + + var genericType = typeof(Dictionary<,>); + frameworkType = genericType.MakeGenericType(new[] { keyType.FrameworkType, valueType.FrameworkType }); + } + } + + public ClickHouseType KeyType => keyType; + + public ClickHouseType ValueType => valueType; + + public override Type FrameworkType => frameworkType; + + public override string Name => "Map"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + var types = node.ChildNodes.Select(parseClickHouseTypeFunc).ToArray(); + var result = new MapType() { UnderlyingTypes = Tuple.Create(types[0], types[1]) }; + return result; + } + + public override object Read(ExtendedBinaryReader reader) + { + var dict = (IDictionary)Activator.CreateInstance(FrameworkType); + + var length = reader.Read7BitEncodedInt(); + + for (var i = 0; i < length; i++) + { + var key = KeyType.Read(reader); // null is not supported as dictionary key in C# + var value = ClearDBNull(ValueType.Read(reader)); + dict.Add(key, value); + } + return dict; + } + + public override string ToString() => $"{Name}({keyType}, {valueType})"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var dict = (IDictionary)value; + writer.Write7BitEncodedInt(dict.Count); + foreach (DictionaryEntry kvp in dict) + { + KeyType.Write(writer, kvp.Key); + ValueType.Write(writer, kvp.Value); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MultiPolygonType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MultiPolygonType.cs new file mode 100644 index 0000000..645a3ba --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/MultiPolygonType.cs @@ -0,0 +1,12 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class MultiPolygonType : ArrayType + { + public MultiPolygonType() + { + UnderlyingType = new PolygonType(); + } + + public override string ToString() => "MultiPolygon"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NestedType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NestedType.cs new file mode 100644 index 0000000..070f543 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NestedType.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class NestedType : TupleType + { + public override string Name => "Nested"; + + public override Type FrameworkType => base.FrameworkType.MakeArrayType(); + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new NestedType + { + UnderlyingTypes = node.ChildNodes.Select(ClearFieldName).Select(parseClickHouseTypeFunc).ToArray(), + }; + } + + // Try to determine if something which is inside a Nested column is a name-type pair + // (param_id UInt8) or a more complex structure (another parameterized type) + // We do not currently support multi-word types + private static SyntaxTreeNode ClearFieldName(SyntaxTreeNode node) + { + if (node.ChildNodes.Count > 0) + return node; + + var name = node.Value; + + var lastSpaceIndex = name.LastIndexOf(' '); + return lastSpaceIndex > 0 ? new SyntaxTreeNode { Value = name.Substring(lastSpaceIndex + 1) } : node; + } + + public override object Read(ExtendedBinaryReader reader) + { + var length = reader.Read7BitEncodedInt(); + var data = Array.CreateInstance(base.FrameworkType, length); + for (var i = 0; i < length; i++) + { + data.SetValue(ClearDBNull(base.Read(reader)), i); + } + return data; + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { + if (value is null || value is DBNull) + { + writer.Write7BitEncodedInt(0); + return; + } + + var collection = (IList)value; + writer.Write7BitEncodedInt(collection.Count); + for (var i = 0; i < collection.Count; i++) + { + base.Write(writer, collection[i]); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NothingType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NothingType.cs new file mode 100644 index 0000000..947dfbc --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NothingType.cs @@ -0,0 +1,16 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class NothingType : ClickHouseType + { + public override Type FrameworkType => typeof(DBNull); + + public override object Read(ExtendedBinaryReader reader) => DBNull.Value; + + public override string ToString() => "Nothing"; + + public override void Write(ExtendedBinaryWriter writer, object value) { } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NullableType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NullableType.cs new file mode 100644 index 0000000..81294a0 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/NullableType.cs @@ -0,0 +1,47 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class NullableType : ParameterizedType + { + public ClickHouseType UnderlyingType { get; set; } + + public override Type FrameworkType + { + get + { + var underlyingFrameworkType = UnderlyingType.FrameworkType; + return underlyingFrameworkType.IsValueType ? typeof(Nullable<>).MakeGenericType(underlyingFrameworkType) : underlyingFrameworkType; + } + } + + public override string Name => "Nullable"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new NullableType + { + UnderlyingType = parseClickHouseTypeFunc(node.SingleChild), + }; + } + + public override object Read(ExtendedBinaryReader reader) => reader.ReadByte() > 0 ? DBNull.Value : UnderlyingType.Read(reader); + + public override string ToString() => $"{Name}({UnderlyingType})"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + if (value == null || value is DBNull) + { + writer.Write((byte)1); + } + else + { + writer.Write((byte)0); + UnderlyingType.Write(writer, value); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ObjectType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ObjectType.cs new file mode 100644 index 0000000..800dd69 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ObjectType.cs @@ -0,0 +1,29 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class ObjectType : ParameterizedType + { + public ClickHouseType UnderlyingType { get; set; } + + public override Type FrameworkType => UnderlyingType.FrameworkType; + + public override string Name => "Object"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new SimpleAggregateFunctionType + { + UnderlyingType = parseClickHouseTypeFunc(node.ChildNodes[0]), + }; + } + + public override object Read(ExtendedBinaryReader reader) => UnderlyingType.Read(reader); + + public override string ToString() => $"{Name}({UnderlyingType})"; + + public override void Write(ExtendedBinaryWriter writer, object value) => UnderlyingType.Write(writer, value); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ParameterizedType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ParameterizedType.cs new file mode 100644 index 0000000..7356e22 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/ParameterizedType.cs @@ -0,0 +1,12 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal abstract class ParameterizedType : ClickHouseType + { + public abstract string Name { get; } + + public abstract ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PointType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PointType.cs new file mode 100644 index 0000000..e8bdf58 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PointType.cs @@ -0,0 +1,21 @@ +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class PointType : TupleType + { + public PointType() + { + UnderlyingTypes = new[] { new Float64Type(), new Float64Type() }; + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { + //if (value is System.Drawing.Point p) + // value = Tuple.Create(p.X, p.Y); + base.Write(writer, value); + } + + public override string ToString() => "Point"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PolygonType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PolygonType.cs new file mode 100644 index 0000000..0d0404f --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/PolygonType.cs @@ -0,0 +1,12 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class PolygonType : ArrayType + { + public PolygonType() + { + UnderlyingType = new RingType(); + } + + public override string ToString() => "Polygon"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/RingType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/RingType.cs new file mode 100644 index 0000000..1a0790e --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/RingType.cs @@ -0,0 +1,12 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class RingType : ArrayType + { + public RingType() + { + UnderlyingType = new PointType(); + } + + public override string ToString() => "Ring"; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/SimpleAggregateFunctionType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/SimpleAggregateFunctionType.cs new file mode 100644 index 0000000..a420b8d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/SimpleAggregateFunctionType.cs @@ -0,0 +1,32 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class SimpleAggregateFunctionType : ParameterizedType + { + public ClickHouseType UnderlyingType { get; set; } + + public string AggregateFunction { get; set; } + + public override Type FrameworkType => UnderlyingType.FrameworkType; + + public override string Name => "SimpleAggregateFunction"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new SimpleAggregateFunctionType + { + AggregateFunction = node.ChildNodes[0].Value, + UnderlyingType = parseClickHouseTypeFunc(node.ChildNodes[1]), + }; + } + + public override object Read(ExtendedBinaryReader reader) => UnderlyingType.Read(reader); + + public override string ToString() => $"{Name}({AggregateFunction}, {UnderlyingType})"; + + public override void Write(ExtendedBinaryWriter writer, object value) => UnderlyingType.Write(writer, value); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/StringType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/StringType.cs new file mode 100644 index 0000000..177bd03 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/StringType.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class StringType : ClickHouseType + { + public override Type FrameworkType => typeof(string); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadString(); + + public override string ToString() => "String"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToString(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TupleType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TupleType.cs new file mode 100644 index 0000000..22f9fb5 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TupleType.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; +using YPermitin.SQLCLR.ClickHouseClient.Utility; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class TupleType : ParameterizedType + { + private Type frameworkType; + private ClickHouseType[] underlyingTypes; + + public ClickHouseType[] UnderlyingTypes + { + get => underlyingTypes; + set + { + underlyingTypes = value; + frameworkType = DeviseFrameworkType(underlyingTypes); + } + } + + private static Type DeviseFrameworkType(ClickHouseType[] underlyingTypes) + { + var count = underlyingTypes.Length; + +#if !NET462 + if (count > 7) + return typeof(LargeTuple); +#endif + + var typeArgs = new Type[count]; + for (var i = 0; i < count; i++) + { + typeArgs[i] = underlyingTypes[i].FrameworkType; + } + var genericType = Type.GetType("System.Tuple`" + typeArgs.Length); + return genericType.MakeGenericType(typeArgs); + } + +#if !NET462 + public ITuple MakeTuple(params object[] values) + { + var count = values.Length; + if (underlyingTypes.Length != count) + throw new ArgumentException($"Count of tuple type elements ({underlyingTypes.Length}) does not match number of elements ({count})"); + + if (count > 7) + return new LargeTuple(values); + + var valuesCopy = new object[count]; + + // Coerce the values into types which can be stored in the tuple + for (int i = 0; i < count; i++) + { + valuesCopy[i] = UnderlyingTypes[i].FrameworkType.IsSubclassOf(typeof(IConvertible)) ? Convert.ChangeType(values[i], UnderlyingTypes[i].FrameworkType, CultureInfo.InvariantCulture) : values[i]; + } + + return (ITuple)Activator.CreateInstance(frameworkType, valuesCopy); + } +#endif + + public override Type FrameworkType => frameworkType; + + public override string Name => "Tuple"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new TupleType + { + UnderlyingTypes = node.ChildNodes.Select(parseClickHouseTypeFunc).ToArray(), + }; + } + + public override string ToString() => $"{Name}({string.Join(",", UnderlyingTypes.Select(t => t.ToString()))})"; + + public override object Read(ExtendedBinaryReader reader) + { + var count = UnderlyingTypes.Length; + var contents = new object[count]; + for (var i = 0; i < count; i++) + { + var value = UnderlyingTypes[i].Read(reader); + contents[i] = ClearDBNull(value); + } +#if !NET462 + return MakeTuple(contents); +#else + return contents; +#endif + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { +#if !NET462 + if (value is ITuple tuple) + { + if (tuple.Length != UnderlyingTypes.Length) + throw new ArgumentException("Wrong number of elements in Tuple", nameof(value)); + for (var i = 0; i < tuple.Length; i++) + { + UnderlyingTypes[i].Write(writer, tuple[i]); + } + return; + } +#endif + if (value is IList list) + { + if (list.Count != UnderlyingTypes.Length) + throw new ArgumentException("Wrong number of elements in Tuple", nameof(value)); + for (var i = 0; i < list.Count; i++) + { + UnderlyingTypes[i].Write(writer, list[i]); + } + return; + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TypeConverter.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TypeConverter.cs new file mode 100644 index 0000000..39cc491 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/TypeConverter.cs @@ -0,0 +1,281 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using YPermitin.SQLCLR.ClickHouseClient.Numerics; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +[assembly: InternalsVisibleTo("ClickHouse.Client.Tests")] // assembly-level tag to expose below classes to tests + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal static class TypeConverter + { + private static readonly IDictionary SimpleTypes = new Dictionary(); + private static readonly IDictionary ParameterizedTypes = new Dictionary(); + private static readonly IDictionary ReverseMapping = new Dictionary(); + + private static readonly IDictionary Aliases = new Dictionary() + { + { "BIGINT", "Int64" }, + { "BIGINT SIGNED", "Int64" }, + { "BIGINT UNSIGNED", "UInt64" }, + { "BINARY", "FixedString" }, + { "BINARY LARGE OBJECT", "String" }, + { "BINARY VARYING", "String" }, + { "BIT", "UInt64" }, + { "BLOB", "String" }, + { "BYTE", "Int8" }, + { "BYTEA", "String" }, + { "CHAR", "String" }, + { "CHAR LARGE OBJECT", "String" }, + { "CHAR VARYING", "String" }, + { "CHARACTER", "String" }, + { "CHARACTER LARGE OBJECT", "String" }, + { "CHARACTER VARYING", "String" }, + { "CLOB", "String" }, + { "DEC", "Decimal" }, + { "DOUBLE", "Float64" }, + { "DOUBLE PRECISION", "Float64" }, + { "ENUM", "Enum" }, + { "FIXED", "Decimal" }, + { "FLOAT", "Float32" }, + { "GEOMETRY", "String" }, + { "INET4", "IPv4" }, + { "INET6", "IPv6" }, + { "INT", "Int32" }, + { "INT SIGNED", "Int32" }, + { "INT UNSIGNED", "UInt32" }, + { "INT1", "Int8" }, + { "INT1 SIGNED", "Int8" }, + { "INT1 UNSIGNED", "UInt8" }, + { "INTEGER", "Int32" }, + { "INTEGER SIGNED", "Int32" }, + { "INTEGER UNSIGNED", "UInt32" }, + { "LONGBLOB", "String" }, + { "LONGTEXT", "String" }, + { "MEDIUMBLOB", "String" }, + { "MEDIUMINT", "Int32" }, + { "MEDIUMINT SIGNED", "Int32" }, + { "MEDIUMINT UNSIGNED", "UInt32" }, + { "MEDIUMTEXT", "String" }, + { "NATIONAL CHAR", "String" }, + { "NATIONAL CHAR VARYING", "String" }, + { "NATIONAL CHARACTER", "String" }, + { "NATIONAL CHARACTER LARGE OBJECT", "String" }, + { "NATIONAL CHARACTER VARYING", "String" }, + { "NCHAR", "String" }, + { "NCHAR LARGE OBJECT", "String" }, + { "NCHAR VARYING", "String" }, + { "NUMERIC", "Decimal" }, + { "NVARCHAR", "String" }, + { "REAL", "Float32" }, + { "SET", "UInt64" }, + { "SINGLE", "Float32" }, + { "SMALLINT", "Int16" }, + { "SMALLINT SIGNED", "Int16" }, + { "SMALLINT UNSIGNED", "UInt16" }, + { "TEXT", "String" }, + { "TIME", "Int64" }, + { "TIMESTAMP", "DateTime" }, + { "TINYBLOB", "String" }, + { "TINYINT", "Int8" }, + { "TINYINT SIGNED", "Int8" }, + { "TINYINT UNSIGNED", "UInt8" }, + { "TINYTEXT", "String" }, + { "VARBINARY", "String" }, + { "VARCHAR", "String" }, + { "VARCHAR2", "String" }, + { "YEAR", "UInt16" }, + { "BOOL", "Bool" }, + { "BOOLEAN", "Bool" }, + { "OBJECT('JSON')", "Json" }, + { "JSON", "Json" }, + }; + + public static IEnumerable RegisteredTypes => SimpleTypes.Keys + .Concat(ParameterizedTypes.Values.Select(t => t.Name)) + .OrderBy(x => x) + .ToArray(); + + static TypeConverter() + { + RegisterPlainType(); + + // Integral types + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + + // Floating point types + RegisterPlainType(); + RegisterPlainType(); + + // Special types + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + + // String types + RegisterPlainType(); + RegisterParameterizedType(); + + // DateTime types + RegisterPlainType(); + RegisterPlainType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + + // Special 'nothing' type + RegisterPlainType(); + + // complex types like Tuple/Array/Nested etc. + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + RegisterParameterizedType(); + + // Geo types + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + RegisterPlainType(); + + // JSON/Object + RegisterParameterizedType(); + + RegisterParameterizedType(); + + // Mapping fixups + ReverseMapping.Add(typeof(ClickHouseDecimal), new Decimal128Type()); + ReverseMapping.Add(typeof(decimal), new Decimal128Type()); +#if NET6_0_OR_GREATER + ReverseMapping.Add(typeof(DateOnly), new DateType()); +#endif + ReverseMapping[typeof(DateTime)] = new DateTimeType(); + ReverseMapping[typeof(DateTimeOffset)] = new DateTimeType(); + } + + private static void RegisterPlainType() + where T : ClickHouseType, new() + { + var type = new T(); + var name = string.Intern(type.ToString()); // There is a limited number of types, interning them will help performance + SimpleTypes.Add(name, type); + if (!ReverseMapping.ContainsKey(type.FrameworkType)) + { + ReverseMapping.Add(type.FrameworkType, type); + } + } + + private static void RegisterParameterizedType() + where T : ParameterizedType, new() + { + var t = new T(); + var name = string.Intern(t.Name); // There is a limited number of types, interning them will help performance + ParameterizedTypes.Add(name, t); + } + + public static ClickHouseType ParseClickHouseType(string type, TypeSettings settings) + { + var node = Parser.Parse(type); + return ParseClickHouseType(node, settings); + } + + internal static ClickHouseType ParseClickHouseType(SyntaxTreeNode node, TypeSettings settings) + { + var typeName = node.Value.Trim().Trim('\''); + + if (Aliases.TryGetValue(typeName.ToUpperInvariant(), out var alias)) + typeName = alias; + + if (typeName.Contains(' ')) + { + var parts = typeName.Split(new[] { " " }, 2, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) + { + typeName = parts[1].Trim(); + } + else + { + throw new ArgumentException($"Cannot parse {node.Value} as type", nameof(node)); + } + } + + if (node.ChildNodes.Count == 0 && SimpleTypes.TryGetValue(typeName, out var typeInfo)) + { + return typeInfo; + } + + if (ParameterizedTypes.ContainsKey(typeName)) + { + return ParameterizedTypes[typeName].Parse(node, (n) => ParseClickHouseType(n, settings), settings); + } + + throw new ArgumentException("Unknown type: " + node.ToString()); + } + + /// + /// Recursively build ClickHouse type from .NET complex type + /// Supports nullable and arrays. + /// + /// framework type to map + /// Corresponding ClickHouse type + public static ClickHouseType ToClickHouseType(Type type) + { + if (ReverseMapping.ContainsKey(type)) + { + return ReverseMapping[type]; + } + + if (type.IsArray) + { + return new ArrayType() { UnderlyingType = ToClickHouseType(type.GetElementType()) }; + } + + var underlyingType = Nullable.GetUnderlyingType(type); + if (underlyingType != null) + { + return new NullableType() { UnderlyingType = ToClickHouseType(underlyingType) }; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition().FullName.StartsWith("System.Tuple", StringComparison.InvariantCulture)) + { + return new TupleType { UnderlyingTypes = type.GetGenericArguments().Select(ToClickHouseType).ToArray() }; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition().FullName.StartsWith("System.Collections.Generic.Dictionary", StringComparison.InvariantCulture)) + { + var types = type.GetGenericArguments().Select(ToClickHouseType).ToArray(); + return new MapType { UnderlyingTypes = Tuple.Create(types[0], types[1]) }; + } + + throw new ArgumentOutOfRangeException(nameof(type), "Unknown type: " + type.ToString()); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt128Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt128Type.cs new file mode 100644 index 0000000..4464640 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt128Type.cs @@ -0,0 +1,11 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt128Type : AbstractBigIntegerType + { + public override int Size => 16; + + public override string ToString() => "UInt128"; + + public override bool Signed => false; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt16Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt16Type.cs new file mode 100644 index 0000000..fd2402f --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt16Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt16Type : IntegerType + { + public override Type FrameworkType => typeof(ushort); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadUInt16(); + + public override string ToString() => "UInt16"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToUInt16(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt256Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt256Type.cs new file mode 100644 index 0000000..8b40030 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt256Type.cs @@ -0,0 +1,11 @@ +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt256Type : AbstractBigIntegerType + { + public override int Size => 32; + + public override string ToString() => "UInt256"; + + public override bool Signed => false; + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt32Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt32Type.cs new file mode 100644 index 0000000..53dd734 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt32Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt32Type : IntegerType + { + public override Type FrameworkType => typeof(uint); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadUInt32(); + + public override string ToString() => "UInt32"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToUInt32(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt64Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt64Type.cs new file mode 100644 index 0000000..dbac009 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt64Type.cs @@ -0,0 +1,18 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt64Type : IntegerType + + { + public override Type FrameworkType => typeof(ulong); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadUInt64(); + + public override string ToString() => "UInt64"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToUInt64(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt8Type.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt8Type.cs new file mode 100644 index 0000000..f9db8b5 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UInt8Type.cs @@ -0,0 +1,17 @@ +using System; +using System.Globalization; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UInt8Type : IntegerType + { + public override Type FrameworkType => typeof(byte); + + public override object Read(ExtendedBinaryReader reader) => reader.ReadByte(); + + public override string ToString() => "UInt8"; + + public override void Write(ExtendedBinaryWriter writer, object value) => writer.Write(Convert.ToByte(value, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UuidType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UuidType.cs new file mode 100644 index 0000000..a648a91 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/UuidType.cs @@ -0,0 +1,37 @@ +using System; +using YPermitin.SQLCLR.ClickHouseClient.Formats; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class UuidType : ClickHouseType + { + public override Type FrameworkType => typeof(Guid); + + public override object Read(ExtendedBinaryReader reader) + { + // Byte manipulation because of ClickHouse's weird GUID/UUID implementation + var bytes = new byte[16]; + reader.Read(bytes, 6, 2); + reader.Read(bytes, 4, 2); + reader.Read(bytes, 0, 4); + reader.Read(bytes, 8, 8); + Array.Reverse(bytes, 8, 8); + return new Guid(bytes); + } + + public override string ToString() => "UUID"; + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var guid = ExtractGuid(value); + var bytes = guid.ToByteArray(); + Array.Reverse(bytes, 8, 8); + writer.Write(bytes, 6, 2); + writer.Write(bytes, 4, 2); + writer.Write(bytes, 0, 4); + writer.Write(bytes, 8, 8); + } + + private static Guid ExtractGuid(object data) => data is Guid g ? g : new Guid((string)data); + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/VariantType.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/VariantType.cs new file mode 100644 index 0000000..d762536 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Types/VariantType.cs @@ -0,0 +1,55 @@ +using System; +using System.Linq; +using YPermitin.SQLCLR.ClickHouseClient.Formats; +using YPermitin.SQLCLR.ClickHouseClient.Types.Grammar; + +namespace YPermitin.SQLCLR.ClickHouseClient.Types +{ + internal class VariantType : ParameterizedType + { + public ClickHouseType[] UnderlyingTypes { get; private set; } + + public override Type FrameworkType => typeof(object); + + public override string Name => "Variant"; + + public override ParameterizedType Parse(SyntaxTreeNode node, Func parseClickHouseTypeFunc, TypeSettings settings) + { + return new VariantType + { + UnderlyingTypes = node.ChildNodes.Select(parseClickHouseTypeFunc).ToArray(), + }; + } + + public override string ToString() => $"{Name}({string.Join(",", UnderlyingTypes.Select(t => t.ToString()))})"; + + public override object Read(ExtendedBinaryReader reader) + { + var typeIndex = reader.ReadByte(); + var type = UnderlyingTypes[typeIndex]; + + return type.Read(reader); + } + + public (int, ClickHouseType) GetMatchingType(object value) + { + var valueType = value?.GetType() ?? typeof(DBNull); + for (int i = 0; i < UnderlyingTypes.Length; i++) + { + var type = UnderlyingTypes[i]; + if (type.FrameworkType == valueType) + { + return (i, type); + } + } + throw new ArgumentException("Could not find matching type for variant", nameof(value)); + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { + var (index, type) = GetMatchingType(value); + writer.Write((byte)index); + type.Write(writer, value); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ClickHouseFeatureMap.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ClickHouseFeatureMap.cs new file mode 100644 index 0000000..f4441ed --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ClickHouseFeatureMap.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using YPermitin.SQLCLR.ClickHouseClient.ADO; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + internal static class ClickHouseFeatureMap + { + private static readonly Dictionary FeatureMap = new(); + + static ClickHouseFeatureMap() + { + var type = typeof(Feature); + var versionsToFeatures = from field in type.GetFields() + let attribute = field.GetCustomAttribute() + where attribute != null + let value = (Feature)field.GetRawConstantValue() + select (value, attribute.Version); + + foreach ((var feature, var version) in versionsToFeatures) + { + if (!FeatureMap.TryAdd(version, feature)) + FeatureMap[version] |= feature; + } + } + + internal static Feature GetFeatureFlags(Version serverVersion) + { + var result = Feature.None; + foreach (var feature in FeatureMap) + { + if (serverVersion >= feature.Key) + result |= feature.Value; + } + return result; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CommandExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CommandExtensions.cs new file mode 100644 index 0000000..ab15274 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CommandExtensions.cs @@ -0,0 +1,27 @@ +using YPermitin.SQLCLR.ClickHouseClient.ADO; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Parameters; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class CommandExtensions + { + public static ClickHouseDbParameter AddParameter(this ClickHouseCommand command, string parameterName, object parameterValue) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = parameterName; + parameter.Value = parameterValue; + command.Parameters.Add(parameter); + return parameter; + } + + public static ClickHouseDbParameter AddParameter(this ClickHouseCommand command, string parameterName, string clickHouseType, object parameterValue) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = parameterName; + parameter.ClickHouseType = clickHouseType; + parameter.Value = parameterValue; + command.Parameters.Add(parameter); + return parameter; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CompressedContent.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CompressedContent.cs new file mode 100644 index 0000000..5b8e98c --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/CompressedContent.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +/// +/// Originally sourced from https://stackoverflow.com/questions/16673714/how-to-compress-http-request-on-the-fly-and-without-loading-compressed-buffer-in +/// +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public class CompressedContent : HttpContent + { + private readonly HttpContent originalContent; + private readonly DecompressionMethods compressionMethod; + + public CompressedContent(HttpContent content, DecompressionMethods compressionMethod) + { + originalContent = content ?? throw new ArgumentNullException(nameof(content)); + this.compressionMethod = compressionMethod; + + if (this.compressionMethod != DecompressionMethods.GZip && this.compressionMethod != DecompressionMethods.Deflate) + { + throw new ArgumentException($"Compression '{compressionMethod}' is not supported. Valid types: GZip, Deflate", nameof(compressionMethod)); + } + + foreach (var header in originalContent.Headers) + { + Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + Headers.ContentEncoding.Add(EnumToLowercaseStringCached.ToString(this.compressionMethod)); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + originalContent?.Dispose(); + } + base.Dispose(disposing); + } + + protected override bool TryComputeLength(out long length) + { + length = -1; + return false; + } + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) + { + using Stream compressedStream = compressionMethod switch + { + DecompressionMethods.GZip => new GZipStream(stream, CompressionLevel.Fastest, leaveOpen: true), + DecompressionMethods.Deflate => new DeflateStream(stream, CompressionMode.Compress, leaveOpen: true), + _ => throw new ArgumentOutOfRangeException(nameof(stream)) + }; + + await originalContent.CopyToAsync(compressedStream).ConfigureAwait(false); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ConnectionExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ConnectionExtensions.cs new file mode 100644 index 0000000..7c09bb8 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/ConnectionExtensions.cs @@ -0,0 +1,42 @@ +using System.Data; +using System.Data.Common; +using System.Threading.Tasks; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Adapters; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class ConnectionExtensions + { + public static Task ExecuteStatementAsync(this DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + return command.ExecuteNonQueryAsync(); + } + + public static Task ExecuteScalarAsync(this DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + return command.ExecuteScalarAsync(); + } + + public static Task ExecuteReaderAsync(this DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + return command.ExecuteReaderAsync(); + } + + public static DataTable ExecuteDataTable(this DbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + using var adapter = new ClickHouseDataAdapter(); + command.CommandText = sql; + adapter.SelectCommand = command; + var dataTable = new DataTable(); + adapter.Fill(dataTable); + return dataTable; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DataReaderExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DataReaderExtensions.cs new file mode 100644 index 0000000..f0f74ee --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DataReaderExtensions.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Data; +using System.Runtime.CompilerServices; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Readers; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class DataReaderExtensions + { + public static string[] GetColumnNames(this IDataReader reader) + { + var count = reader.FieldCount; + var names = new string[count]; + for (int i = 0; i < count; i++) + { + names[i] = reader.GetName(i); + } + + return names; + } + + internal static ClickHouseType[] GetClickHouseColumnTypes(this ClickHouseDataReader reader) + { + var count = reader.FieldCount; + var names = new ClickHouseType[count]; + for (int i = 0; i < count; i++) + { + names[i] = reader.GetClickHouseType(i); + } + + return names; + } + + internal static IEnumerable AsEnumerable(this IDataReader reader) + { + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + yield return values; + } + } + +#if !NET462 + internal static IEnumerable AsEnumerable(this ITuple tuple) + { + for (int i = 0; i < tuple.Length; i++) + { + yield return tuple[i]; + } + } +#endif + } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DictionaryExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DictionaryExtensions.cs new file mode 100644 index 0000000..0f0922d --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/DictionaryExtensions.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class DictionaryExtensions + { + public static bool TryAdd(this IDictionary dictionary, TKey key, TValue value) + { + if (dictionary.ContainsKey(key)) + return false; + dictionary.Add(key, value); + return true; + } + + public static void Set(this IDictionary dictionary, TKey key, TValue value) + { + if (dictionary.ContainsKey(key)) + dictionary[key] = value; + else + dictionary.Add(key, value); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumToLowercaseStringCached.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumToLowercaseStringCached.cs new file mode 100644 index 0000000..f1bf70a --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumToLowercaseStringCached.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Concurrent; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + internal static class EnumToLowercaseStringCached + where T : Enum + { + private static readonly ConcurrentDictionary Values = new ConcurrentDictionary(); + + public static string ToString(T value) + { + return Values.GetOrAdd(value, (v) => v.ToString().ToLowerInvariant()); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumerableExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumerableExtensions.cs new file mode 100644 index 0000000..4d03c49 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/EnumerableExtensions.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +//using System.Buffers; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class EnumerableExtensions + { + public static void Deconstruct(this IList list, out T first, out T second) + { + if (list.Count != 2) + throw new ArgumentException($"Expected 2 elements in list, got {list.Count}"); + first = list[0]; + second = list[1]; + } + + public static void Deconstruct(this IList list, out T first, out T second, out T third) + { + if (list.Count != 3) + throw new ArgumentException($"Expected 3 elements in list, got {list.Count}"); + first = list[0]; + second = list[1]; + third = list[2]; + } + + public static IEnumerable<(T[], int)> BatchRented(this IEnumerable enumerable, int batchSize) + { + List items = new List(); + + //var array = ArrayPool.Shared.Rent(batchSize); + int counter = 0; + + foreach (var item in enumerable) + { + //array[counter++] = item; + counter++; + items.Add(item); + + if (counter >= batchSize) + { + yield return (items.ToArray(), counter); + //yield return (array, counter); + counter = 0; + //array = ArrayPool.Shared.Rent(batchSize); + items = new List(); + } + } + if (counter > 0) + //yield return (array, counter); + yield return (items.ToArray(), counter); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/IsExternalInit.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/IsExternalInit.cs new file mode 100644 index 0000000..e76b433 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/IsExternalInit.cs @@ -0,0 +1,8 @@ +// Helper definition to workaround VS2019 bug +// CS0518 Predefined type 'System.Runtime.CompilerServices.IsExternalInit' is not defined or imported +// See: https://stackoverflow.com/a/64749403/ +// ReSharper disable once CheckNamespace +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit { } +} diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/LargeTuple.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/LargeTuple.cs new file mode 100644 index 0000000..2091fd9 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/LargeTuple.cs @@ -0,0 +1,20 @@ +#if !NET462 +using System.Runtime.CompilerServices; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + internal class LargeTuple : ITuple + { + private readonly object[] items; + + public LargeTuple(params object[] items) + { + this.items = items; + } + + public object this[int index] => items[index]; + + public int Length => items.Length; + } +} +#endif \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/MathUtils.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/MathUtils.cs new file mode 100644 index 0000000..16fd738 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/MathUtils.cs @@ -0,0 +1,42 @@ +using System; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class MathUtils + { + public static long ToPower(int value, int power) + { + checked + { + long result = 1; + while (power > 0) + { + if ((power & 1) == 1) + { + result *= value; + } + + power >>= 1; + if (power <= 0) + { + break; + } + + value *= value; + } + return result; + } + } + + public static long ShiftDecimalPlaces(long value, int places) + { + if (places == 0) + { + return value; + } + + var factor = ToPower(10, Math.Abs(places)); + return places < 0 ? value / factor : value * factor; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/NameValueCollectionExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/NameValueCollectionExtensions.cs new file mode 100644 index 0000000..373d421 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/NameValueCollectionExtensions.cs @@ -0,0 +1,19 @@ +using System.Collections.Specialized; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class NameValueCollectionExtensions + { + public static void SetOrRemove(this NameValueCollection collection, string name, string value) + { + if (!string.IsNullOrEmpty(value)) + { + collection.Set(name, value); + } + else + { + collection.Remove(name); + } + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SchemaDescriber.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SchemaDescriber.cs new file mode 100644 index 0000000..dfc2467 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SchemaDescriber.cs @@ -0,0 +1,107 @@ +using System; +using System.Data; +using System.Linq; +using System.Text; +using YPermitin.SQLCLR.ClickHouseClient.ADO; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Adapters; +using YPermitin.SQLCLR.ClickHouseClient.ADO.Readers; +using YPermitin.SQLCLR.ClickHouseClient.Types; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + internal static class SchemaDescriber + { + public static DataTable DescribeSchema(this ClickHouseDataReader reader) + { + var table = new DataTable(); + table.Columns.Add("ColumnName", typeof(string)); + table.Columns.Add("ColumnOrdinal", typeof(int)); + table.Columns.Add("ColumnSize", typeof(int)); + table.Columns.Add("NumericPrecision", typeof(int)); + table.Columns.Add("NumericScale", typeof(int)); + table.Columns.Add("IsUnique", typeof(bool)); + table.Columns.Add("IsKey", typeof(bool)); + table.Columns.Add("DataType", typeof(Type)); + table.Columns.Add("AllowDBNull", typeof(bool)); + table.Columns.Add("ProviderType", typeof(string)); + table.Columns.Add("IsAliased", typeof(bool)); + table.Columns.Add("IsExpression", typeof(bool)); + table.Columns.Add("IsIdentity", typeof(bool)); + table.Columns.Add("IsAutoIncrement", typeof(bool)); + table.Columns.Add("IsRowVersion", typeof(bool)); + table.Columns.Add("IsHidden", typeof(bool)); + table.Columns.Add("IsLong", typeof(bool)); + table.Columns.Add("IsReadOnly", typeof(bool)); + + for (int ordinal = 0; ordinal < reader.FieldCount; ordinal++) + { + var chType = reader.GetClickHouseType(ordinal); + + var row = table.NewRow(); + row["ColumnName"] = reader.GetName(ordinal); + row["ColumnOrdinal"] = ordinal; + row["ColumnSize"] = -1; + row["DataType"] = chType is NullableType nt ? nt.UnderlyingType.FrameworkType : chType.FrameworkType; + row["ProviderType"] = chType; + row["IsLong"] = chType is StringType; + row["AllowDBNull"] = chType is NullableType; + row["IsReadOnly"] = true; + row["IsRowVersion"] = false; + row["IsUnique"] = false; + row["IsKey"] = false; + row["IsAutoIncrement"] = false; + + if (chType is DecimalType dt) + { + row["ColumnSize"] = dt.Size; + row["NumericPrecision"] = dt.Precision; + row["NumericScale"] = dt.Scale; + } + table.Rows.Add(row); + } + return table; + } + + public static DataTable DescribeSchema(this ClickHouseConnection connection, string type, string[] restrictions) => type switch + { + "Columns" => DescribeColumns(connection, restrictions), + _ => throw new NotSupportedException(), + }; + + private static DataTable DescribeColumns(ClickHouseConnection connection, string[] restrictions) + { + var command = connection.CreateCommand(); + var query = new StringBuilder("SELECT database as Database, table as Table, name as Name, type as ProviderType, type as DataType FROM system.columns"); + var database = restrictions != null && restrictions.Length > 0 ? restrictions[0] : null; + var table = restrictions != null && restrictions.Length > 1 ? restrictions[1] : null; + + if (database != null) + { + query.Append(" WHERE database={database:String}"); + command.AddParameter("database", "String", database); + } + + if (table != null) + { + query.Append(" AND table={table:String}"); + command.AddParameter("table", "String", table); + } + + command.CommandText = query.ToString(); + using var adapter = new ClickHouseDataAdapter(); + adapter.SelectCommand = command; + var result = new DataTable(); + adapter.Fill(result); + + foreach (var row in result.Rows.Cast()) + { + var clickHouseType = TypeConverter.ParseClickHouseType((string)row["ProviderType"], TypeSettings.Default); + row["ProviderType"] = clickHouseType.ToString(); + // TODO: this should return actual framework type like other implementations do + row["DataType"] = clickHouseType.FrameworkType.ToString().Replace("System.", string.Empty); + } + + return result; + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SinceVersionAttribute.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SinceVersionAttribute.cs new file mode 100644 index 0000000..cf32058 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/SinceVersionAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + [AttributeUsage(AttributeTargets.Field)] + internal class SinceVersionAttribute : Attribute + { + public SinceVersionAttribute(string version) => Version = Version.Parse(version); + + public Version Version { get; } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/StringExtensions.cs b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/StringExtensions.cs new file mode 100644 index 0000000..643f027 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/Utility/StringExtensions.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; + +namespace YPermitin.SQLCLR.ClickHouseClient.Utility +{ + public static class StringExtensions + { + public static string Escape(this string str) => str.Replace("\\", "\\\\").Replace("\'", "\\\'").Replace("\n", "\\n").Replace("\t", "\\t"); + + public static string QuoteSingle(this string str) => str.StartsWith("'", StringComparison.InvariantCulture) && str.EndsWith("'", StringComparison.InvariantCulture) ? str : $"'{str}'"; + + public static string QuoteDouble(this string str) => str.StartsWith("\"", StringComparison.InvariantCulture) && str.EndsWith("\"", StringComparison.InvariantCulture) ? str : $"\"{str}\""; + + /// + /// Encloses column name in backticks (`). Escapes ` symbol if met inside name + /// Does nothing if column is already enclosed + /// + /// Column name + /// Backticked column name + public static string EncloseColumnName(this string str) + { + if (string.IsNullOrEmpty(str)) + return str; + if (str[0] == '`' && str[str.Length - 1] == '`') + return str; // Early return if already enclosed + + var builder = new StringBuilder(); + builder.Append('`'); + builder.Append(str.Replace("`", "\\`")); + builder.Append('`'); + return builder.ToString(); + } + + public static string ToSnakeCase(this string str) + { + var result = new StringBuilder(); + for (int i = 0; i < str.Length; i++) + { + if (char.IsUpper(str[i]) && i > 0) + { + result.Append('_'); + } + result.Append(char.ToLower(str[i], System.Globalization.CultureInfo.InvariantCulture)); + } + + return result.ToString(); + } + + public static string ReplaceMultipleWords(this string input, Dictionary replacements) + { + if (replacements == null || replacements.Count == 0) + return input; + var regex = "(" + string.Join("\\b|", replacements.Keys) + "\\b)"; + return Regex.Replace(input, regex, (Match m) => { return replacements[m.Value]; }); + } + } +} \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/packages.config b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/packages.config new file mode 100644 index 0000000..2becba0 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Libs/ClickHouseClient/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/ClickHouseClient/Readme.md b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Readme.md new file mode 100644 index 0000000..8bd0fb0 --- /dev/null +++ b/SQL-Server-SQLCLR/Projects/ClickHouseClient/Readme.md @@ -0,0 +1,313 @@ +# ClickHouseClient + +Расширение SQL CLR для SQL Server для работы с СУБД ClickHouse из T-SQL в качестве клиента. + +## Собранное решение + +Собранную DLL для установки расширения SQLCLR можно скачать в разделе [релизы](https://github.com/YPermitin/SQLServerTools/releases). + +## Обратная связь и новости + +Вопросы, предложения и любую другую информацию [отправляйте на электронную почту](mailto:i.need.ypermitin@yandex.ru). + +Новости по проектам или новым материалам в [Telegram-канале](https://t.me/TinyDevVault). + +## Функциональность + + + +## Окружение для разработки + +Для окружение разработчика необходимы: + +* [.NET Framework 4.8 SDK](https://support.microsoft.com/ru-ru/topic/microsoft-net-framework-4-8-автономный-установщик-для-windows-9d23f658-3b97-68ab-d013-aa3c3e7495e0) +* [.NET 6 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) +* [Visual Studio 2022](https://visualstudio.microsoft.com/ru/vs/) +* [Microsoft SQL Server 2012+](https://www.microsoft.com/ru-ru/sql-server/sql-server-downloads) + +## Состав проекта + +Проекты и библиотеки в составе решения: + +* **Apps** - библиотеки и вспомогательные проекты. + + * **ClickHouseClient.CLI** - консольное приложение для отладки использования расширения SQLCLR и демонстрации вызова методов расширения. + +* **Libs** - библиотеки и вспомогательные проекты. + + * **ClickHouseClient** - клиентская библиотека [ClickHouse.Client от Oleg V. Kozlyuk](https://github.com/DarkWanderer/ClickHouse.Client), адаптированная под .NET Framework 4.8 для возможности запуска в среде .NET Framework, совместимой с SQLCLR. + * **ClickHouseClient.Entry** - библиотека расширение SQLCLR для работы с ClickHouse из T-SQL. + +## Установка + +Для установки на стороне SQL Server нужно выполнить несколько шагов: + +1. Собрать проект **ClickHouseClient.Entry** в режиме **Release**. +2. Полученную DLL **ClickHouseClient.Entry.dll** и ВСЕ другие файлы в каталоге сборки скопировать на сервер, где установлен экземпляр SQL Server. Пусть для примера путь к DLL на сервере будет **"C:\Share\SQLCLR\ClickHouseClient.Entry.dll"**. Там же в каталоге будут файлы **ClickHouseClient.dll**, **Newtonsoft.Json.dll**, **NodaTime.dll**. +3. Выбрать базу для установки. Например, пусть она называется **PowerSQLCLR**. + +4. [Включим интеграцию с CLR](https://learn.microsoft.com/en-us/sql/relational-databases/clr-integration/clr-integration-enabling?view=sql-server-ver16), а также настроим права, установим зависимости от глобальных сборок из GAC .NET Framwork, а после создадим объекты процедур и функций для работы с расширениями. + +```sql +-- Этап 1. Включаем поддержку SQLCLR для инстанса SQL Server и доверие для базы данных. +EXEC sp_configure 'clr enabled', 1; +RECONFIGURE; +GO +ALTER DATABASE [PowerSQLCLR] SET TRUSTWORTHY ON; +GO + +-- Этап 2. Подготавливаем сертификаты Microsoft для сборок .NET Framework. +-- Этот шаг необходим для подключения стандартных сборок .NET +-- к инстансу SQL Server. Для добавленного сертификата создаем +-- служебную учетную запись и разрешаем работать со сборками. +USE [master]; + +CREATE CERTIFICATE [MS.NETcer] +FROM EXECUTABLE FILE = + 'C:\Windows\Microsoft.NET\Framework64\v4.0.30319\System.Net.Http.dll'; +GO +CREATE LOGIN [MS.NETcer] FROM CERTIFICATE [MS.NETcer]; +GO +GRANT UNSAFE ASSEMBLY TO [MS.NETcer]; +GO +DENY CONNECT SQL TO [MS.NETcer] +GO +ALTER LOGIN [MS.NETcer] DISABLE +GO + +-- Этап 3. Добавляем стандартные сборки .NET Framework в служебную базу. +-- Эти сборки необходимы для работы клиента ClickHouse. +USE [PowerSQLCLR]; + +CREATE ASSEMBLY [System.Net.Http] +FROM 'C:\Windows\Microsoft.NET\Framework64\v4.0.30319\System.Net.Http.dll' +WITH PERMISSION_SET = UNSAFE; +GO + +CREATE ASSEMBLY [System.Web] +FROM 'C:\Windows\Microsoft.NET\Framework64\v4.0.30319\System.Web.dll' +WITH PERMISSION_SET = UNSAFE; +GO + +-- Этап 4. Удаляем объекты расширения SQLCLR клиента ClickHouse, +-- если они уже существуют. Ниже они будут созданы заново. +USE [PowerSQLCLR]; +DROP FUNCTION IF EXISTS [dbo].[fn_CHExecuteScalar]; +DROP FUNCTION IF EXISTS [dbo].[fn_CHExecuteSimple]; +DROP FUNCTION IF EXISTS [dbo].[fn_CHGetCreateTempDbTableCommand]; +DROP PROCEDURE IF EXISTS [dbo].[sp_CHExecuteToTempTable]; +DROP PROCEDURE IF EXISTS [dbo].[sp_CHExecuteToGlobalTempTable]; +DROP PROCEDURE IF EXISTS [dbo].[sp_CHExecuteStatement]; +DROP PROCEDURE IF EXISTS [dbo].[sp_CHExecuteBulkInsertFromTempTable]; +DROP ASSEMBLY IF EXISTS [ClickHouseClient.Entry]; +DROP ASSEMBLY IF EXISTS [ClickHouseClient]; +GO + +-- Этап 5. Создаем сборку клиента ClickHouse и расширения SQLCLR в служебной базе, +-- а также все объекты для работы с ней. +-- ВНИМАНИЕ!!! Путь к файлу DLL нужно актуализировать под ваше окружение. +USE [PowerSQLCLR]; + +CREATE ASSEMBLY [ClickHouseClient] + FROM 'C:\Share\SQLCLR\ClickHouseClient.dll' + WITH PERMISSION_SET = UNSAFE; +GO + +CREATE ASSEMBLY [ClickHouseClient.Entry] + FROM 'C:\Share\SQLCLR\ClickHouseClient.Entry.dll' + WITH PERMISSION_SET = UNSAFE; +GO + +CREATE FUNCTION [fn_CHExecuteScalar]( + @connectionString nvarchar(max), + @queryText nvarchar(max) +) +RETURNS nvarchar(max) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteScalar]; +GO + +CREATE FUNCTION [dbo].[fn_CHExecuteSimple]( + @connectionString nvarchar(max), + @queryText nvarchar(max) +) +RETURNS TABLE ( + [ResultValue] nvarchar(max) +) +AS +EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteSimple]; +GO + +CREATE FUNCTION [fn_CHGetCreateTempDbTableCommand]( + @connectionString nvarchar(max), + @queryText nvarchar(max), + @tempTableName nvarchar(max) +) +RETURNS nvarchar(max) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[GetCreateTempDbTableCommand]; +GO + +CREATE PROCEDURE [dbo].[sp_CHExecuteStatement] +( + @connectionString nvarchar(max), + @queryText nvarchar(max) +) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteStatement]; +GO + +CREATE PROCEDURE [dbo].[sp_CHExecuteToTempTable] +( + @connectionString nvarchar(max), + @queryText nvarchar(max), + @tempTableName nvarchar(max) +) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteToTempTable]; +GO + +CREATE PROCEDURE [dbo].[sp_CHExecuteToGlobalTempTable] +( + @connectionString nvarchar(max), + @queryText nvarchar(max), + @tempTableName nvarchar(max), + @sqlServerConnectionString nvarchar(max) +) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteToGlobalTempTable]; +GO + +CREATE PROCEDURE [dbo].[sp_CHExecuteBulkInsertFromTempTable] +( + @connectionString nvarchar(max), + @sourceTempTableName nvarchar(max), + @destinationTableName nvarchar(max) +) +AS EXTERNAL NAME [ClickHouseClient.Entry].[YPermitin.SQLCLR.ClickHouseClient.Entry.EntryClickHouseClient].[ExecuteBulkInsertFromTempTable]; +GO +``` + +После все будет готово для использования расширения. + +## Примеры работы + +Несколько примеров работы с расширением после установки из T-SQL. + +* Получаем версию ClickHouse. + +```sql +SELECT [PowerSQLCLR].[dbo].[fn_CHExecuteScalar]( + -- Строка подключения + 'Host=yy-comp;Port=8123;Username=default;password=;Database=default;', + -- текст запроса + 'select version()') + +-- Пример результата: +-- 25.2.1.2434 +``` + +* Пример выполнения простого запроса, который возвращает одну колоноку. Для возврата нескольких колонок используется кортеж, сериализованный в JSON. В T-SQL полученные элементы JSON парсятся конструкциями SQL Server. + +```sql +select + JSON_VALUE(d.ResultValue, '$.Item1') AS [DatabaseName], + JSON_VALUE(d.ResultValue, '$.Item2') [Engine], + JSON_VALUE(d.ResultValue, '$.Item3') AS [DataPath], + CAST(JSON_VALUE(d.ResultValue, '$.Item4') AS uniqueidentifier) AS [UUID] +from [PowerSQLCLR].[dbo].fn_CHExecuteSimple( + -- Строка подключения + 'Host=yy-comp;Port=8123;Username=default;password=;Database=default;', + -- Запрос + ' +SELECT + tuple(name, engine, data_path,uuid) +FROM `system`.`databases` +' +) d +``` + +* Создаем временную таблицу и сохраняем в нее результат запроса. + +```sql +IF(OBJECT_ID('tempdb..#logs') IS NOT NULL) + DROP TABLE #logs; +CREATE TABLE #logs +( + [EventTime] datetime2(0), + [Query] nvarchar(max), + [Tables] nvarchar(max), + [QueryId] uniqueidentifier +); + +EXECUTE [PowerSQLCLR].[dbo].[sp_CHExecuteToTempTable] + -- Строка подключения + 'Host=yy-comp;Port=8123;Username=default;password=;Database=default;', + -- Текст запроса + ' +select + event_time, + query, + tables, + query_id +from `system`.query_log +limit 1000 +', + -- Имя временной таблицы для сохранения результата + '#logs'; + +-- Считываем результат +SELECT * FROM #logs +``` + +* Создаем ГЛОБАЛЬНУЮ временную таблицу и сохраняем в нее результат запроса. + +```sql +IF(OBJECT_ID('tempdb..##logs') IS NOT NULL) + DROP TABLE ##logs; +CREATE TABLE ##logs +( + [EventTime] datetime2(0), + [Query] nvarchar(max), + [Tables] nvarchar(max), + [QueryId] uniqueidentifier +); + +EXECUTE [PowerSQLCLR].[dbo].[sp_CHExecuteToGlobalTempTable] + -- Строка подключения + 'Host=yy-comp;Port=8123;Username=default;password=;Database=default;', + -- Текст запроса + ' +select + event_time, + query, + tables, + query_id +from `system`.query_log +limit 1000 +', + -- Имя временной таблицы для сохранения результата + '##logs', + -- Строка подключения к SQL Server для BULK INSERT. + -- Строка контекстного подключения для этого не подходит. + 'server=localhost;database=master;trusted_connection=true'; + +-- Считываем результат +SELECT * FROM ##logs +``` + +При использовании глобальной таблицы можно достичь более высокой производительности за счет переноса в нее данных через BULK INSERT (если указана строка подключения к SQL Server). + +* Запуск произвольной команды без возвращения результата. + +```sql +EXECUTE [PowerSQLCLR].[dbo].[sp_CHExecuteStatement] + -- Строка подключения + 'Host=yy-comp;Port=8123;Username=default;password=;Database=default;', + -- Запрос + ' +CREATE TABLE IF NOT EXISTS SimpleTable +( + Id UInt64, + Period datetime DEFAULT now(), + Name String +) +ENGINE = MergeTree +ORDER BY Id; +' +``` \ No newline at end of file diff --git a/SQL-Server-SQLCLR/Projects/Readme.md b/SQL-Server-SQLCLR/Projects/Readme.md index ab2ebe6..eb571c9 100644 --- a/SQL-Server-SQLCLR/Projects/Readme.md +++ b/SQL-Server-SQLCLR/Projects/Readme.md @@ -8,7 +8,8 @@ | Название | Описание | | -------- | -------- | -| [HttpHelper](HttpHelper/) | Расширение для работы с HTTP-запросами из T-SQL. | -| [YellowMetadataReader](YellowMetadataReader/) | Расширение для работы с метаданными баз 1С напрямую из T-SQL. | -| [YellowInfobaseWrapperMananger](YellowInfobaseWrapperMananger) | Расширение для создания оберточных баз для баз 1С, чтобы была возможность работать с базами в режиме "только для чтения". | -| [DevAdmHelpers](DevAdmHelpers) | Набор вспомогательных возможностей для администраторов и разработчиков SQL Server. | +| [HttpHelper](HttpHelper/Readme.md) | Расширение для работы с HTTP-запросами из T-SQL. | +| [YellowMetadataReader](YellowMetadataReader/Readme.md) | Расширение для работы с метаданными баз 1С напрямую из T-SQL. | +| [YellowInfobaseWrapperMananger](YellowInfobaseWrapperMananger/Readme.md) | Расширение для создания оберточных баз для баз 1С, чтобы была возможность работать с базами в режиме "только для чтения". | +| [DevAdmHelpers](DevAdmHelpers/Readme.md) | Набор вспомогательных возможностей для администраторов и разработчиков SQL Server. | +| [ClickHouseClient](ClickHouseClient/Readme.md) | Расширение клиента для работы с СУБД ClickHouse из T-SQL. |