From 214005e990fc4fa2cfd67dd4e813cd24774c1ffb Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 4 Dec 2024 22:08:43 +0530 Subject: [PATCH 1/3] chore: clickhouse sql support for v3 --- .../app/clickhouseReader/reader.go | 19 ++++++++++++++++++- .../app/clickhouseReader/wrapper.go | 5 +++++ pkg/query-service/app/querier/querier.go | 16 ++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 256ab98d68..4b511a7d92 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -4038,9 +4038,26 @@ func (r *ClickHouseReader) GetListResultV3(ctx context.Context, query string) ([ var t time.Time for idx, v := range vars { if columnNames[idx] == "timestamp" { - t = time.Unix(0, int64(*v.(*uint64))) + switch v := v.(type) { + case *uint64: + t = time.Unix(0, int64(*v)) + case *time.Time: + t = *v + } } else if columnNames[idx] == "timestamp_datetime" { t = *v.(*time.Time) + } else if columnNames[idx] == "events" { + var events []map[string]interface{} + eventsFromDB, ok := v.(*[]string) + if !ok { + continue + } + for _, event := range *eventsFromDB { + var eventMap map[string]interface{} + json.Unmarshal([]byte(event), &eventMap) + events = append(events, eventMap) + } + row[columnNames[idx]] = events } else { row[columnNames[idx]] = v } diff --git a/pkg/query-service/app/clickhouseReader/wrapper.go b/pkg/query-service/app/clickhouseReader/wrapper.go index fc0a71dc8d..735cafae2b 100644 --- a/pkg/query-service/app/clickhouseReader/wrapper.go +++ b/pkg/query-service/app/clickhouseReader/wrapper.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "regexp" + "strings" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" @@ -44,6 +45,10 @@ func (c clickhouseConnWrapper) addClickHouseSettings(ctx context.Context, query settings["log_comment"] = logComment } + if strings.Contains(query, "signoz_traces") { + settings["max_result_rows"] = 100000 + } + if c.settings.MaxBytesToRead != "" { settings["max_bytes_to_read"] = c.settings.MaxBytesToRead } diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 9ef0e2582c..2e9e387988 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -465,7 +465,15 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan } } - queries, err := q.builder.PrepareQueries(params) + queries := make(map[string]string) + var err error + if params.CompositeQuery.QueryType == v3.QueryTypeBuilder { + queries, err = q.builder.PrepareQueries(params) + } else if params.CompositeQuery.QueryType == v3.QueryTypeClickHouseSQL { + for name, chQuery := range params.CompositeQuery.ClickHouseQueries { + queries[name] = chQuery.Query + } + } if err != nil { return nil, nil, err @@ -534,7 +542,11 @@ func (q *querier) QueryRange(ctx context.Context, params *v3.QueryRangeParamsV3) case v3.QueryTypePromQL: results, errQueriesByName, err = q.runPromQueries(ctx, params) case v3.QueryTypeClickHouseSQL: - results, errQueriesByName, err = q.runClickHouseQueries(ctx, params) + if params.CompositeQuery.PanelType == v3.PanelTypeList || params.CompositeQuery.PanelType == v3.PanelTypeTrace { + results, errQueriesByName, err = q.runBuilderListQueries(ctx, params) + } else { + results, errQueriesByName, err = q.runClickHouseQueries(ctx, params) + } default: err = fmt.Errorf("invalid query type") } From 528c7c6155c627f9f5594e62f17d74a64b8fcfe3 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 16 Jan 2025 18:13:52 +0530 Subject: [PATCH 2/3] chore: add limit --- pkg/query-service/app/clickhouseReader/reader.go | 1 + pkg/query-service/app/clickhouseReader/wrapper.go | 6 +++--- pkg/query-service/app/querier/querier.go | 1 + pkg/query-service/app/querier/v2/querier.go | 1 + pkg/query-service/constants/constants.go | 2 ++ 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 86dde1b6dd..0a8a5acead 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -218,6 +218,7 @@ func NewReaderFromClickhouseConnection( MaxBytesToRead: os.Getenv("ClickHouseMaxBytesToRead"), OptimizeReadInOrderRegex: os.Getenv("ClickHouseOptimizeReadInOrderRegex"), OptimizeReadInOrderRegexCompiled: regexCompiled, + MaxResultRowsForCHQuery: constants.MaxResultRowsForCHQuery, }, } diff --git a/pkg/query-service/app/clickhouseReader/wrapper.go b/pkg/query-service/app/clickhouseReader/wrapper.go index 735cafae2b..0a2476e785 100644 --- a/pkg/query-service/app/clickhouseReader/wrapper.go +++ b/pkg/query-service/app/clickhouseReader/wrapper.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "regexp" - "strings" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" @@ -18,6 +17,7 @@ type ClickhouseQuerySettings struct { MaxBytesToRead string OptimizeReadInOrderRegex string OptimizeReadInOrderRegexCompiled *regexp.Regexp + MaxResultRowsForCHQuery int } type clickhouseConnWrapper struct { @@ -45,8 +45,8 @@ func (c clickhouseConnWrapper) addClickHouseSettings(ctx context.Context, query settings["log_comment"] = logComment } - if strings.Contains(query, "signoz_traces") { - settings["max_result_rows"] = 100000 + if ctx.Value("enfore_max_result_rows") != nil { + settings["max_result_rows"] = c.settings.MaxResultRowsForCHQuery } if c.settings.MaxBytesToRead != "" { diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 2e9e387988..bfacaca402 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -542,6 +542,7 @@ func (q *querier) QueryRange(ctx context.Context, params *v3.QueryRangeParamsV3) case v3.QueryTypePromQL: results, errQueriesByName, err = q.runPromQueries(ctx, params) case v3.QueryTypeClickHouseSQL: + ctx = context.WithValue(ctx, "enfore_max_result_rows", true) if params.CompositeQuery.PanelType == v3.PanelTypeList || params.CompositeQuery.PanelType == v3.PanelTypeTrace { results, errQueriesByName, err = q.runBuilderListQueries(ctx, params) } else { diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 86a5af334a..0837ddb5a4 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -548,6 +548,7 @@ func (q *querier) QueryRange(ctx context.Context, params *v3.QueryRangeParamsV3) if params.CompositeQuery.PanelType == v3.PanelTypeList || params.CompositeQuery.PanelType == v3.PanelTypeTrace { results, errQueriesByName, err = q.runBuilderListQueries(ctx, params) } else { + ctx = context.WithValue(ctx, "enfore_max_result_rows", true) results, errQueriesByName, err = q.runClickHouseQueries(ctx, params) } default: diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 242b2cd4a3..8f4eb436b4 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -734,3 +734,5 @@ func init() { } const TRACE_V4_MAX_PAGINATION_LIMIT = 10000 + +const MaxResultRowsForCHQuery = 1_000_000 From 9dc277acba53f778161c1183bcb5b83634b4d6e5 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 16 Jan 2025 18:19:35 +0530 Subject: [PATCH 3/3] chore: typo --- pkg/query-service/app/clickhouseReader/wrapper.go | 2 +- pkg/query-service/app/querier/querier.go | 2 +- pkg/query-service/app/querier/v2/querier.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/wrapper.go b/pkg/query-service/app/clickhouseReader/wrapper.go index 0a2476e785..b56ebd1589 100644 --- a/pkg/query-service/app/clickhouseReader/wrapper.go +++ b/pkg/query-service/app/clickhouseReader/wrapper.go @@ -45,7 +45,7 @@ func (c clickhouseConnWrapper) addClickHouseSettings(ctx context.Context, query settings["log_comment"] = logComment } - if ctx.Value("enfore_max_result_rows") != nil { + if ctx.Value("enforce_max_result_rows") != nil { settings["max_result_rows"] = c.settings.MaxResultRowsForCHQuery } diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index bfacaca402..d7c1e8b48d 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -542,7 +542,7 @@ func (q *querier) QueryRange(ctx context.Context, params *v3.QueryRangeParamsV3) case v3.QueryTypePromQL: results, errQueriesByName, err = q.runPromQueries(ctx, params) case v3.QueryTypeClickHouseSQL: - ctx = context.WithValue(ctx, "enfore_max_result_rows", true) + ctx = context.WithValue(ctx, "enforce_max_result_rows", true) if params.CompositeQuery.PanelType == v3.PanelTypeList || params.CompositeQuery.PanelType == v3.PanelTypeTrace { results, errQueriesByName, err = q.runBuilderListQueries(ctx, params) } else { diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 0837ddb5a4..dcfe62c1ed 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -548,7 +548,7 @@ func (q *querier) QueryRange(ctx context.Context, params *v3.QueryRangeParamsV3) if params.CompositeQuery.PanelType == v3.PanelTypeList || params.CompositeQuery.PanelType == v3.PanelTypeTrace { results, errQueriesByName, err = q.runBuilderListQueries(ctx, params) } else { - ctx = context.WithValue(ctx, "enfore_max_result_rows", true) + ctx = context.WithValue(ctx, "enforce_max_result_rows", true) results, errQueriesByName, err = q.runClickHouseQueries(ctx, params) } default: