From 5cd941f628617f4b67413ec26e6a2eef1db583cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20H=C3=A4nsel?= <haensel@creations.de>
Date: Tue, 21 Jan 2025 16:47:01 +0100
Subject: [PATCH] chore(outputs.sql): Make default create table template
 suitable for ClickHouse

This changes the default TableTemplate of the SQL output plugin so that
it can be used with ClickHouse. Previously it was invalid for ClickHouse
because it didn't include a PRIMARY KEY or ORDER BY clause, which is
required for ClickHouse.

This then also allows to remove the custom TableTemplate from the
ClickHouse unit test.

This also adds another placeholder {TAG_COLUMNS} that allows to get a
list of all tag columns in a TableTemplate. This can then be used to use
the tag columns as a primary or sort key, especially in ClickHouse.
---
 plugins/outputs/sql/README.md   |  1 +
 plugins/outputs/sql/sample.conf |  1 +
 plugins/outputs/sql/sql.go      | 11 ++++++++++-
 plugins/outputs/sql/sql_test.go |  1 -
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md
index 74214c55a7832..ea04000c64f14 100644
--- a/plugins/outputs/sql/README.md
+++ b/plugins/outputs/sql/README.md
@@ -99,6 +99,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf
index f193d2b2e2dbf..7da6e2d216ae9 100644
--- a/plugins/outputs/sql/sample.conf
+++ b/plugins/outputs/sql/sample.conf
@@ -18,6 +18,7 @@
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go
index 6574c76402c95..cc7389d46b24a 100644
--- a/plugins/outputs/sql/sql.go
+++ b/plugins/outputs/sql/sql.go
@@ -146,6 +146,7 @@ func (p *SQL) deriveDatatype(value interface{}) string {
 
 func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	columns := make([]string, 0, len(metric.TagList())+len(metric.FieldList())+1)
+	tagColumns := make([]string, 0, len(metric.TagList()))
 
 	if p.TimestampColumn != "" {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(p.TimestampColumn), p.Convert.Timestamp))
@@ -153,6 +154,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 
 	for _, tag := range metric.TagList() {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(tag.Key), p.Convert.Text))
+		tagColumns = append(tagColumns, quoteIdent(tag.Key))
 	}
 
 	var datatype string
@@ -165,6 +167,13 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	query = strings.ReplaceAll(query, "{TABLE}", quoteIdent(metric.Name()))
 	query = strings.ReplaceAll(query, "{TABLELITERAL}", quoteStr(metric.Name()))
 	query = strings.ReplaceAll(query, "{COLUMNS}", strings.Join(columns, ","))
+	query = strings.ReplaceAll(query, "{TAG_COLUMNS}", strings.Join(tagColumns, ","))
+
+	if p.Driver == "clickhouse" {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", fmt.Sprintf("ORDER BY (%s, %s)", strings.Join(tagColumns, ","), p.TimestampColumn))
+	} else {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", "")
+	}
 
 	return query
 }
@@ -273,7 +282,7 @@ func init() {
 
 func newSQL() *SQL {
 	return &SQL{
-		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS})",
+		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS}) {SORT_KEY_CLAUSE}",
 		TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1",
 		TimestampColumn:     "timestamp",
 		Convert: ConvertStruct{
diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go
index a0e59ff533900..5c7bde8adec04 100644
--- a/plugins/outputs/sql/sql_test.go
+++ b/plugins/outputs/sql/sql_test.go
@@ -368,7 +368,6 @@ func TestClickHouseIntegration(t *testing.T) {
 	p.Log = testutil.Logger{}
 	p.Driver = "clickhouse"
 	p.DataSourceName = address
-	p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp"
 	p.Convert.Integer = "Int64"
 	p.Convert.Text = "String"
 	p.Convert.Timestamp = "DateTime"