diff --git a/CHANGELOG.md b/CHANGELOG.md index 819ad1005c..7055a5b022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Main (unreleased) - (_Experimental_) Improve parsing of truncated queries in `database_observability.mysql` (@cristiangreco) +- (_Experimental_) Capture schema name for query samples in `database_observability.mysql` (@cristiangreco) + - Add json format support for log export via faro receiver (@ravishankar15) - Add livedebugging support for `prometheus.remote_write` (@ravishankar15) diff --git a/internal/component/database_observability/mysql/collector/query_sample.go b/internal/component/database_observability/mysql/collector/query_sample.go index 5252ed2a76..2a47ab5100 100644 --- a/internal/component/database_observability/mysql/collector/query_sample.go +++ b/internal/component/database_observability/mysql/collector/query_sample.go @@ -26,6 +26,7 @@ const ( const selectQuerySamples = ` SELECT digest, + schema_name, query_sample_text, query_sample_seen, query_sample_timer_wait @@ -127,8 +128,8 @@ func (c *QuerySample) fetchQuerySamples(ctx context.Context) error { break } - var digest, sampleText, sampleSeen, sampleTimerWait string - err := rs.Scan(&digest, &sampleText, &sampleSeen, &sampleTimerWait) + var digest, schemaName, sampleText, sampleSeen, sampleTimerWait string + err := rs.Scan(&digest, &schemaName, &sampleText, &sampleSeen, &sampleTimerWait) if err != nil { level.Error(c.logger).Log("msg", "failed to scan result set for query samples", "err", err) continue @@ -142,20 +143,20 @@ func (c *QuerySample) fetchQuerySamples(ctx context.Context) error { sampleText = sampleText[:idx] } } else { - level.Debug(c.logger).Log("msg", "skipping parsing truncated query", "digest", digest) + level.Debug(c.logger).Log("msg", "skipping parsing truncated query", "schema", schemaName, "digest", digest) continue } } stmt, err := sqlparser.Parse(sampleText) if err != nil { - level.Error(c.logger).Log("msg", "failed to parse sql query", "digest", digest, "err", err) + level.Error(c.logger).Log("msg", "failed to parse sql query", "schema", schemaName, "digest", digest, "err", err) continue } sampleRedactedText, err := sqlparser.RedactSQLQuery(sampleText) if err != nil { - level.Error(c.logger).Log("msg", "failed to redact sql query", "digest", digest, "err", err) + level.Error(c.logger).Log("msg", "failed to redact sql query", "schema", schemaName, "digest", digest, "err", err) continue } @@ -164,8 +165,8 @@ func (c *QuerySample) fetchQuerySamples(ctx context.Context) error { Entry: logproto.Entry{ Timestamp: time.Unix(0, time.Now().UnixNano()), Line: fmt.Sprintf( - `level=info msg="query samples fetched" op="%s" instance="%s" digest="%s" query_type="%s" query_sample_seen="%s" query_sample_timer_wait="%s" query_sample_redacted="%s"`, - OP_QUERY_SAMPLE, c.instanceKey, digest, c.stmtType(stmt), sampleSeen, sampleTimerWait, sampleRedactedText, + `level=info msg="query samples fetched" op="%s" instance="%s" schema="%s" digest="%s" query_type="%s" query_sample_seen="%s" query_sample_timer_wait="%s" query_sample_redacted="%s"`, + OP_QUERY_SAMPLE, c.instanceKey, schemaName, digest, c.stmtType(stmt), sampleSeen, sampleTimerWait, sampleRedactedText, ), }, } @@ -177,8 +178,8 @@ func (c *QuerySample) fetchQuerySamples(ctx context.Context) error { Entry: logproto.Entry{ Timestamp: time.Unix(0, time.Now().UnixNano()), Line: fmt.Sprintf( - `level=info msg="table name parsed" op="%s" instance="%s" digest="%s" table="%s"`, - OP_QUERY_PARSED_TABLE_NAME, c.instanceKey, digest, table, + `level=info msg="table name parsed" op="%s" instance="%s" schema="%s" digest="%s" table="%s"`, + OP_QUERY_PARSED_TABLE_NAME, c.instanceKey, schemaName, digest, table, ), }, } diff --git a/internal/component/database_observability/mysql/collector/query_sample_test.go b/internal/component/database_observability/mysql/collector/query_sample_test.go index 96dc30eff1..030d88b796 100644 --- a/internal/component/database_observability/mysql/collector/query_sample_test.go +++ b/internal/component/database_observability/mysql/collector/query_sample_test.go @@ -29,72 +29,78 @@ func TestQuerySample(t *testing.T) { name: "select query", rows: [][]driver.Value{{ "abc123", + "some_schema", "select * from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "insert query", rows: [][]driver.Value{{ "abc123", + "some_schema", "insert into some_table (`id`, `name`) values (1, 'foo')", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="insert" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="insert into some_table(id, name) values (:redacted1, :redacted2)"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="insert" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="insert into some_table(id, name) values (:redacted1, :redacted2)"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "update query", rows: [][]driver.Value{{ "abc123", + "some_schema", "update some_table set active=false, reason=null where id = 1 and name = 'foo'", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="update" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="update some_table set active = false, reason = null where id = :redacted1 and name = :redacted2"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="update" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="update some_table set active = false, reason = null where id = :redacted1 and name = :redacted2"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "delete query", rows: [][]driver.Value{{ "abc123", + "some_schema", "delete from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="delete" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="delete from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="delete" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="delete from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "join two tables", rows: [][]driver.Value{{ "abc123", + "some_schema", "select t.id, t.val1, o.val2 FROM some_table t inner join other_table as o on t.id = o.id where o.val2 = 1 order by t.val1 desc", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select t.id, t.val1, o.val2 from some_table as t join other_table as o on t.id = o.id where o.val2 = :redacted1 order by t.val1 desc"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="other_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select t.id, t.val1, o.val2 from some_table as t join other_table as o on t.id = o.id where o.val2 = :redacted1 order by t.val1 desc"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="other_table"`, }, }, { name: "subquery", rows: [][]driver.Value{{ "abc123", + "some_schema", `select ifnull(schema_name, 'none') as schema_name, digest, count_star from (select * from performance_schema.events_statements_summary_by_digest where schema_name not in ('mysql', 'performance_schema', 'information_schema') and last_seen > date_sub(now(), interval 86400 second) order by last_seen desc)q @@ -103,110 +109,141 @@ func TestQuerySample(t *testing.T) { "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" ` + + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" ` + `query_sample_redacted="select ifnull(schema_name, :redacted1) as schema_name, digest, count_star from (select * from ` + `performance_schema.events_statements_summary_by_digest where schema_name not in ::redacted2 ` + `and last_seen > date_sub(now(), interval :redacted3 second) order by last_seen desc) as q ` + `group by q.schema_name, q.digest, q.count_star limit :redacted4"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="performance_schema.events_statements_summary_by_digest"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="performance_schema.events_statements_summary_by_digest"`, }, }, { name: "with comment", rows: [][]driver.Value{{ "abc123", + "some_schema", "select val1, /* val2,*/ val3 from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select val1, val3 from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select val1, val3 from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "truncated query", rows: [][]driver.Value{{ "xyz456", + "some_schema", "insert into some_table (`id1`, `id2`, `id3`, `id...", "2024-02-02T00:00:00.000Z", "2000", }, { "abc123", + "some_schema", "select * from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "truncated in multi-line comment", rows: [][]driver.Value{{ "abc123", + "some_schema", "select * from some_table where id = 1 /*traceparent='00-abc...", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, }, }, { name: "start transaction", rows: [][]driver.Value{{ "abc123", + "some_schema", "START TRANSACTION", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="begin"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="begin"`, }, }, { name: "commit", rows: [][]driver.Value{{ "abc123", + "some_schema", "COMMIT", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="commit"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="commit"`, }, }, { name: "alter table", rows: [][]driver.Value{{ "abc123", + "some_schema", "alter table some_table modify enumerable enum('val1', 'val2') not null", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="alter table some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="alter table some_table"`, }, }, { name: "sql parse error", rows: [][]driver.Value{{ "xyz456", + "some_schema", "not valid sql", "2024-02-02T00:00:00.000Z", "2000", }, { "abc123", + "some_schema", "select * from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", }}, logs: []string{ - `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, - `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, + }, + }, + { + name: "multiple schemas", + rows: [][]driver.Value{{ + "abc123", + "some_schema", + "select * from some_table where id = 1", + "2024-01-01T00:00:00.000Z", + "1000", + }, { + "abc123", + "other_schema", + "select * from some_table where id = 1", + "2024-01-01T00:00:00.000Z", + "1000", + }}, + logs: []string{ + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, + `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="other_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, + `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="other_schema" digest="abc123" table="some_table"`, }, }, } @@ -233,6 +270,7 @@ func TestQuerySample(t *testing.T) { WillReturnRows( sqlmock.NewRows([]string{ "digest", + "schema_name", "query_sample_text", "query_sample_seen", "query_sample_timer_wait", @@ -330,11 +368,13 @@ func TestQuerySampleSQLDriverErrors(t *testing.T) { WillReturnRows( sqlmock.NewRows([]string{ "digest", + "schema_name", "query_sample_text", "query_sample_seen", "query_sample_timer_wait", }).AddRow( "abc123", + "some_schema", "select * from some_table where id = 1", "2024-01-01T00:00:00.000Z", "1000", @@ -356,8 +396,8 @@ func TestQuerySampleSQLDriverErrors(t *testing.T) { require.Equal(t, model.LabelSet{"job": database_observability.JobName}, entry.Labels) } - require.Equal(t, `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, lokiEntries[0].Line) - require.Equal(t, `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" digest="abc123" table="some_table"`, lokiEntries[1].Line) + require.Equal(t, `level=info msg="query samples fetched" op="query_sample" instance="mysql-db" schema="some_schema" digest="abc123" query_type="select" query_sample_seen="2024-01-01T00:00:00.000Z" query_sample_timer_wait="1000" query_sample_redacted="select * from some_table where id = :redacted1"`, lokiEntries[0].Line) + require.Equal(t, `level=info msg="table name parsed" op="query_parsed_table_name" instance="mysql-db" schema="some_schema" digest="abc123" table="some_table"`, lokiEntries[1].Line) err = mock.ExpectationsWereMet() require.NoError(t, err)